[test-runner] Introduce mapbox-gl-native test expectations

The `mapbox-gl-native`-specific render tests expected results will
be stored at the `render-test/expected/` subfolder.

Test runner will automatically put new expected results there if
invoked with `UPDATE=true`.

6 files changed, 72 insertions, 17 deletions

diff --git a/render-test/main.cpp b/render-test/main.cpp
index 35f0cdea30..db141649f2 100644
--- a/render-test/main.cpp
+++ b/render-test/main.cpp
@@ -7,6 +7,7 @@
 #include "runner.hpp"
 
 #include <random>
+#include <regex>
 
 #define ANSI_COLOR_RED        "\x1b[31m"
 #define ANSI_COLOR_GREEN      "\x1b[32m"
@@ -18,6 +19,25 @@
 #define ANSI_COLOR_LIGHT_GRAY "\x1b[90m"
 #define ANSI_COLOR_RESET      "\x1b[0m"
 
+namespace {
+
+TestPaths makeTestPaths(mbgl::filesystem::path stylePath) {
+    auto defaultExpectations = stylePath;
+    defaultExpectations.remove_filename();
+    const static std::regex regex{ TestRunner::getBasePath() };
+    auto platformExpectations = std::regex_replace(defaultExpectations.string(), regex, TestRunner::getPlatformExpectationsPath());
+    assert(!defaultExpectations.empty());
+    assert(!platformExpectations.empty());
+
+    return {
+        std::move(stylePath),
+        std::move(defaultExpectations),
+        std::move(platformExpectations)
+    };
+}
+
+} // namespace
+
 int main(int argc, char** argv) {
     bool recycleMap;
     bool shuffle;
@@ -31,11 +51,12 @@ int main(int argc, char** argv) {
     const auto ignores = parseIgnores();
 
     // Recursively traverse through the test paths and collect test directories containing "style.json".
-    std::vector<mbgl::filesystem::path> testPaths;
+    std::vector<TestPaths> testPaths;
+    testPaths.reserve(ids.size());
     for (const auto& id : ids) {
         for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(mbgl::filesystem::path(id))) {
             if (testPath.path().filename() == "style.json") {
-                testPaths.push_back(testPath);
+                testPaths.emplace_back(makeTestPaths(testPath));
             }
         }
     }
@@ -67,7 +88,7 @@ int main(int argc, char** argv) {
         std::string& status = metadata.status;
         std::string& color = metadata.color;
 
-        id = testPath.remove_filename().string();
+        id = testPath.defaultExpectations.string();
         id = id.substr(rootLength + 1, id.length() - rootLength - 2);
 
         bool shouldIgnore = false;
diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp
index 4be83a5436..2f568359ab 100644
--- a/render-test/metadata.hpp
+++ b/render-test/metadata.hpp
@@ -17,10 +17,16 @@ struct TestStatistics {
     uint32_t passedTests = 0;
 };
 
+struct TestPaths {
+    mbgl::filesystem::path stylePath;
+    mbgl::filesystem::path defaultExpectations;
+    mbgl::filesystem::path platformExpectations;
+};
+
 struct TestMetadata {
     TestMetadata() = default;
 
-    mbgl::filesystem::path path;
+    TestPaths paths;
     mbgl::JSDocument document;
 
     mbgl::Size size{ 512u, 512u };
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index fd048f08c6..49f084465c 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -15,6 +15,7 @@
 
 #include "parser.hpp"
 #include "metadata.hpp"
+#include "runner.hpp"
 
 #include <sstream>
 #include <regex>
@@ -243,17 +244,18 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
 
     std::vector<std::string> ids;
     for (const auto& id : args::get(testNameValues)) {
-        ids.emplace_back(testDefaultPath + "/" + id);
+        ids.emplace_back(TestRunner::getBasePath() + "/" + id);
     }
 
     if (ids.empty()) {
-        ids.emplace_back(testDefaultPath);
+        ids.emplace_back(TestRunner::getBasePath());
     }
 
     return ArgumentsTuple {
         recycleMapFlag ? args::get(recycleMapFlag) : false,
         shuffleFlag ? args::get(shuffleFlag) : false, seedValue ? args::get(seedValue) : 1u,
-        testPathValue ? args::get(testPathValue) : testDefaultPath, ids
+        testPathValue ? args::get(testPathValue) : testDefaultPath,
+        std::move(ids)
     };
 }
 
@@ -288,13 +290,13 @@ std::vector<std::pair<std::string, std::string>> parseIgnores() {
     return ignores;
 }
 
-TestMetadata parseTestMetadata(const mbgl::filesystem::path& path) {
+TestMetadata parseTestMetadata(const TestPaths& paths) {
     TestMetadata metadata;
-    metadata.path = path;
+    metadata.paths = paths;
 
-    auto maybeJson = readJson(path.string());
+    auto maybeJson = readJson(paths.stylePath.string());
     if (!maybeJson.is<mbgl::JSDocument>()) { // NOLINT
-        metadata.errorMessage = std::string("Unable to parse: ") + path.string();
+        metadata.errorMessage = std::string("Unable to parse: ") + metadata.paths.stylePath.string();
         return metadata;
     }
 
@@ -303,14 +305,14 @@ TestMetadata parseTestMetadata(const mbgl::filesystem::path& path) {
 
     if (!metadata.document.HasMember("metadata")) {
         mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s",
-                           path.c_str());
+                           paths.stylePath.c_str());
         return metadata;
     }
 
     const mbgl::JSValue& metadataValue = metadata.document["metadata"];
     if (!metadataValue.HasMember("test")) {
         mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s",
-                           path.c_str());
+                           paths.stylePath.c_str());
         return metadata;
     }
 
diff --git a/render-test/parser.hpp b/render-test/parser.hpp
index be98719ab5..3e69968152 100644
--- a/render-test/parser.hpp
+++ b/render-test/parser.hpp
@@ -11,6 +11,7 @@
 
 struct TestMetadata;
 struct TestStatistics;
+struct TestPaths;
 
 using ErrorMessage = std::string;
 using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
@@ -24,7 +25,8 @@ std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base)
 
 ArgumentsTuple parseArguments(int argc, char** argv);
 std::vector<std::pair<std::string, std::string>> parseIgnores();
-TestMetadata parseTestMetadata(const mbgl::filesystem::path& path);
+
+TestMetadata parseTestMetadata(const TestPaths& paths);
 
 std::string createResultPage(const TestStatistics&, const std::vector<TestMetadata>&, bool shuffle, uint32_t seed);
 
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 5bd855c43a..7b0f210b62 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -26,8 +26,24 @@
 #include <cassert>
 #include <regex>
 
+
+// static 
+const std::string& TestRunner::getBasePath() {
+    const static std::string result =
+        std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration/render-tests");
+    return result;
+}
+
+// static
+const std::string& TestRunner::getPlatformExpectationsPath() {
+    const static std::string result =
+        std::string(TEST_RUNNER_ROOT_PATH).append("/render-test/expected");
+    return result;
+}
+
 bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& metadata) {
-    const std::string base = metadata.path.remove_filename().string();
+    const std::string& base = metadata.paths.defaultExpectations.string();
+
     metadata.actual = mbgl::encodePNG(actual);
 
     if (actual.size.isEmpty()) {
@@ -37,7 +53,8 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met
 
 #if !TEST_READ_ONLY
     if (getenv("UPDATE")) {
-        mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actual));
+        mbgl::filesystem::create_directories(metadata.paths.platformExpectations);
+        mbgl::util::write_file(metadata.paths.platformExpectations.string() + "/expected.png", mbgl::encodePNG(actual));
         return true;
     }
 
@@ -48,8 +65,10 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met
     mbgl::PremultipliedImage diff { actual.size };
 
     double pixels = 0.0;
+    const auto& expectedPath = mbgl::filesystem::exists(metadata.paths.platformExpectations) ?
+        metadata.paths.platformExpectations : metadata.paths.defaultExpectations;
 
-    for (const auto& entry: readExpectedEntries(base)) {
+    for (const auto& entry: readExpectedEntries(expectedPath)) {
         mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
         if (!maybeExpectedImage) {
             metadata.errorMessage = "Failed to load expected image " + entry;
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index cbc0f42546..920a9ee372 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -14,6 +14,11 @@ public:
     bool run(TestMetadata&);
     void reset();
 
+    /// Returns path of the render tests root directory.
+    static const std::string& getBasePath();
+    /// Returns path of mapbox-gl-native expectations directory.
+    static const std::string& getPlatformExpectationsPath();
+
 private:
     bool runOperations(const std::string& key, TestMetadata&);
     bool checkImage(mbgl::PremultipliedImage&& image, TestMetadata&);