Support continuous map rendering mode in test runner

 - Output image is not generated in continous mode

4 files changed, 99 insertions, 66 deletions

diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp
index 1d5a346f22..cb21d3e804 100644
--- a/render-test/metadata.hpp
+++ b/render-test/metadata.hpp
@@ -90,6 +90,7 @@ struct TestMetadata {
     TestPaths paths;
     mbgl::JSDocument document;
     bool renderTest = true;
+    bool outputsImage = true;
 
     mbgl::Size size{ 512u, 512u };
     float pixelRatio = 1.0f;
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index da5e6bea81..02eabe5a60 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -607,8 +607,21 @@ TestMetadata parseTestMetadata(const TestPaths& paths) {
     }
 
     if (testValue.HasMember("mapMode")) {
+        metadata.outputsImage = true;
         assert(testValue["mapMode"].IsString());
-        metadata.mapMode = testValue["mapMode"].GetString() == std::string("tile") ? mbgl::MapMode::Tile : mbgl::MapMode::Static;
+        std::string mapModeStr = testValue["mapMode"].GetString();
+        if (mapModeStr == "tile")
+            metadata.mapMode = mbgl::MapMode::Tile;
+        else if (mapModeStr == "continuous") {
+            metadata.mapMode = mbgl::MapMode::Continuous;
+            metadata.outputsImage = false;
+        } else if (mapModeStr == "static")
+            metadata.mapMode = mbgl::MapMode::Static;
+        else {
+            mbgl::Log::Warning(
+                mbgl::Event::ParseStyle, "Unknown map mode: %s. Falling back to static mode", mapModeStr.c_str());
+            metadata.mapMode = mbgl::MapMode::Static;
+        }
     }
 
     // Test operations handled in runner.cpp.
@@ -703,19 +716,21 @@ std::string createResultItem(const TestMetadata& metadata, bool hasFailedTests)
     html.append("<div class=\"test " + metadata.status + (shouldHide ? " hide" : "") + "\">\n");
     html.append(R"(<h2><span class="label" style="background: )" + metadata.color + "\">" + metadata.status + "</span> " + metadata.id + "</h2>\n");
     if (metadata.status != "errored") {
-        if (metadata.renderTest) {
-            html.append("<img width=" + mbgl::util::toString(metadata.size.width));
-            html.append(" height=" + mbgl::util::toString(metadata.size.height));
-            html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.actual) + "\"");
-            html.append(" data-alt-src=\"data:image/png;base64," + encodeBase64(metadata.expected) + "\">\n");
-
-            html.append("<img width=" + mbgl::util::toString(metadata.size.width));
-            html.append(" height=" + mbgl::util::toString(metadata.size.height));
-            html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.diff) + "\">\n");
-        } else {
-            html.append("<img width=" + mbgl::util::toString(metadata.size.width));
-            html.append(" height=" + mbgl::util::toString(metadata.size.height));
-            html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.actual) + "\">\n");
+        if (metadata.outputsImage) {
+            if (metadata.renderTest) {
+                html.append("<img width=" + mbgl::util::toString(metadata.size.width));
+                html.append(" height=" + mbgl::util::toString(metadata.size.height));
+                html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.actual) + "\"");
+                html.append(" data-alt-src=\"data:image/png;base64," + encodeBase64(metadata.expected) + "\">\n");
+
+                html.append("<img width=" + mbgl::util::toString(metadata.size.width));
+                html.append(" height=" + mbgl::util::toString(metadata.size.height));
+                html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.diff) + "\">\n");
+            } else {
+                html.append("<img width=" + mbgl::util::toString(metadata.size.width));
+                html.append(" height=" + mbgl::util::toString(metadata.size.height));
+                html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.actual) + "\">\n");
+            }
         }
     } else {
         assert(!metadata.errorMessage.empty());
diff --git a/render-test/render_test.cpp b/render-test/render_test.cpp
index c3a9c6dd77..df1658265a 100644
--- a/render-test/render_test.cpp
+++ b/render-test/render_test.cpp
@@ -100,7 +100,8 @@ int runRenderTests(int argc, char** argv) {
             errored = !runner.run(metadata) || !metadata.errorMessage.empty();
         }
 
-        bool passed = !errored && !metadata.diff.empty() && metadata.difference <= metadata.allowed;
+        bool passed =
+            !errored && (!metadata.outputsImage || !metadata.diff.empty()) && metadata.difference <= metadata.allowed;
 
         if (shouldIgnore) {
             if (passed) {
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index bf31793b27..96d0031148 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -165,81 +165,97 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage,
     const std::string& base = metadata.paths.defaultExpectations();
     const std::vector<mbgl::filesystem::path>& expectations = metadata.paths.expectations;
 
-    metadata.actual = mbgl::encodePNG(actualImage);
+    if (metadata.outputsImage) {
+        metadata.actual = mbgl::encodePNG(actualImage);
 
-    if (actualImage.size.isEmpty()) {
-        metadata.errorMessage = "Invalid size for actual image";
-        return false;
-    }
+        if (actualImage.size.isEmpty()) {
+            metadata.errorMessage = "Invalid size for actual image";
+            return false;
+        }
 
 #if !TEST_READ_ONLY
-    if (getenv("UPDATE_PLATFORM")) {
-        mbgl::filesystem::create_directories(expectations.back());
-        mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage));
-        return true;
-    } else if (getenv("UPDATE_DEFAULT")) {
-        mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage));
-        return true;
-    } else if (getenv("UPDATE_METRICS")) {
-        if (!metadata.metrics.isEmpty()) {
+        if (getenv("UPDATE_PLATFORM")) {
             mbgl::filesystem::create_directories(expectations.back());
-            mbgl::util::write_file(expectations.back().string() + "/metrics.json", serializeMetrics(metadata.metrics));
+            mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage));
+            return true;
+        } else if (getenv("UPDATE_DEFAULT")) {
+            mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage));
             return true;
         }
-    }
 
-    mbgl::util::write_file(base + "/actual.png", metadata.actual);
+        mbgl::util::write_file(base + "/actual.png", metadata.actual);
 #endif
 
-    mbgl::PremultipliedImage expectedImage { actualImage.size };
-    mbgl::PremultipliedImage imageDiff { actualImage.size };
+        mbgl::PremultipliedImage expectedImage{actualImage.size};
+        mbgl::PremultipliedImage imageDiff{actualImage.size};
 
-    double pixels = 0.0;
-    std::vector<std::string> expectedImagesPaths;
-    mbgl::filesystem::path expectedMetricsPath;
-    for (auto rit = expectations.rbegin(); rit!= expectations.rend(); ++rit) {
-        if (mbgl::filesystem::exists(*rit)) {
-            if (metadata.expectedMetrics.isEmpty()) {
-                mbgl::filesystem::path maybeExpectedMetricsPath{ *rit };
-                maybeExpectedMetricsPath.replace_filename("metrics.json");
-                metadata.expectedMetrics = readExpectedMetrics(maybeExpectedMetricsPath);
+        double pixels = 0.0;
+        std::vector<std::string> expectedImagesPaths;
+        for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) {
+            if (mbgl::filesystem::exists(*rit)) {
+                expectedImagesPaths = readExpectedImageEntries(*rit);
+                if (!expectedImagesPaths.empty()) break;
             }
-            expectedImagesPaths = readExpectedImageEntries(*rit);
-            if (!expectedImagesPaths.empty()) break;
         }
-    }
 
-    if (expectedImagesPaths.empty()) {
-        metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string();
-        return false;
-    }
-    
-    for (const auto& entry: expectedImagesPaths) {
-        mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
-        if (!maybeExpectedImage) {
-            metadata.errorMessage = "Failed to load expected image " + entry;
+        if (expectedImagesPaths.empty()) {
+            metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string();
             return false;
         }
 
-        metadata.expected = *maybeExpectedImage;
+        for (const auto& entry : expectedImagesPaths) {
+            mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
+            if (!maybeExpectedImage) {
+                metadata.errorMessage = "Failed to load expected image " + entry;
+                return false;
+            }
 
-        expectedImage = mbgl::decodeImage(*maybeExpectedImage);
+            metadata.expected = *maybeExpectedImage;
 
-        pixels = // implicitly converting from uint64_t
-            mapbox::pixelmatch(actualImage.data.get(), expectedImage.data.get(), expectedImage.size.width,
-                               expectedImage.size.height, imageDiff.data.get(), 0.1285); // Defined in GL JS
+            expectedImage = mbgl::decodeImage(*maybeExpectedImage);
 
-        metadata.diff = mbgl::encodePNG(imageDiff);
+            pixels = // implicitly converting from uint64_t
+                mapbox::pixelmatch(actualImage.data.get(),
+                                   expectedImage.data.get(),
+                                   expectedImage.size.width,
+                                   expectedImage.size.height,
+                                   imageDiff.data.get(),
+                                   0.1285); // Defined in GL JS
+
+            metadata.diff = mbgl::encodePNG(imageDiff);
 
 #if !TEST_READ_ONLY
-        mbgl::util::write_file(base + "/diff.png", metadata.diff);
+            mbgl::util::write_file(base + "/diff.png", metadata.diff);
 #endif
 
-        metadata.difference = pixels / expectedImage.size.area();
-        if (metadata.difference <= metadata.allowed) {
-            break;
+            metadata.difference = pixels / expectedImage.size.area();
+            if (metadata.difference <= metadata.allowed) {
+                break;
+            }
+        }
+    }
+
+#if !TEST_READ_ONLY
+    if (getenv("UPDATE_METRICS")) {
+        if (!metadata.metrics.isEmpty()) {
+            mbgl::filesystem::create_directories(expectations.back());
+            mbgl::util::write_file(expectations.back().string() + "/metrics.json", serializeMetrics(metadata.metrics));
+            return true;
         }
     }
+#endif
+
+    mbgl::filesystem::path expectedMetricsPath;
+    for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) {
+        if (mbgl::filesystem::exists(*rit)) {
+            if (metadata.expectedMetrics.isEmpty()) {
+                mbgl::filesystem::path maybeExpectedMetricsPath{*rit};
+                maybeExpectedMetricsPath.replace_filename("metrics.json");
+                metadata.expectedMetrics = readExpectedMetrics(maybeExpectedMetricsPath);
+            }
+        }
+    }
+
     // Check file size metrics.
     for (const auto& expected : metadata.expectedMetrics.fileSize) {
         auto actual = metadata.metrics.fileSize.find(expected.first);
@@ -870,7 +886,7 @@ bool TestRunner::run(TestMetadata& metadata) {
 
     mbgl::PremultipliedImage image;
     try {
-        image = frontend.render(map);
+        if (metadata.outputsImage) image = frontend.render(map);
     } catch (const std::exception&) {
         return false;
     }