[test runner] Expectations paths are represented with std::vectorupstream/mikhail_render_test_expectations

4 files changed, 40 insertions, 22 deletions

diff --git a/render-test/main.cpp b/render-test/main.cpp
index db141649f2..ef9352d21d 100644
--- a/render-test/main.cpp
+++ b/render-test/main.cpp
@@ -22,17 +22,18 @@
 namespace {
 
 TestPaths makeTestPaths(mbgl::filesystem::path stylePath) {
-    auto defaultExpectations = stylePath;
-    defaultExpectations.remove_filename();
-    const static std::regex regex{ TestRunner::getBasePath() };
-    auto platformExpectations = std::regex_replace(defaultExpectations.string(), regex, TestRunner::getPlatformExpectationsPath());
-    assert(!defaultExpectations.empty());
-    assert(!platformExpectations.empty());
+    std::vector<mbgl::filesystem::path> expectations{ stylePath };
+    expectations.front().remove_filename();
+
+    const static std::regex regex{ TestRunner::getBasePath() };    
+    for (const std::string& path : TestRunner::getPlatformExpectationsPaths()) {
+        expectations.emplace_back(std::regex_replace(expectations.front().string(), regex, path));
+        assert(!expectations.back().empty());
+    }
 
     return {
         std::move(stylePath),
-        std::move(defaultExpectations),
-        std::move(platformExpectations)
+        std::move(expectations)
     };
 }
 
@@ -88,7 +89,7 @@ int main(int argc, char** argv) {
         std::string& status = metadata.status;
         std::string& color = metadata.color;
 
-        id = testPath.defaultExpectations.string();
+        id = testPath.defaultExpectations();
         id = id.substr(rootLength + 1, id.length() - rootLength - 2);
 
         bool shouldIgnore = false;
diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp
index 2f568359ab..98e208fe54 100644
--- a/render-test/metadata.hpp
+++ b/render-test/metadata.hpp
@@ -19,8 +19,12 @@ struct TestStatistics {
 
 struct TestPaths {
     mbgl::filesystem::path stylePath;
-    mbgl::filesystem::path defaultExpectations;
-    mbgl::filesystem::path platformExpectations;
+    std::vector<mbgl::filesystem::path> expectations;
+
+    std::string defaultExpectations() const {
+        assert(!expectations.empty());
+        return expectations.front().string();
+    }
 };
 
 struct TestMetadata {
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 1224d398ab..192978ffa5 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -35,14 +35,17 @@ const std::string& TestRunner::getBasePath() {
 }
 
 // static
-const std::string& TestRunner::getPlatformExpectationsPath() {
-    const static std::string result =
-        std::string(TEST_RUNNER_ROOT_PATH).append("/render-test/expected");
+const std::vector<std::string>& TestRunner::getPlatformExpectationsPaths() {
+    // TODO: Populate from command line.
+    const static std::vector<std::string> result {
+        std::string(TEST_RUNNER_ROOT_PATH).append("/render-test/expected")
+    };
     return result;
 }
 
 bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& metadata) {
-    const std::string& base = metadata.paths.defaultExpectations.string();
+    const std::string& base = metadata.paths.defaultExpectations();
+    const std::vector<mbgl::filesystem::path>& expectations = metadata.paths.expectations;
 
     metadata.actual = mbgl::encodePNG(actual);
 
@@ -52,11 +55,11 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met
     }
 
 #if !TEST_READ_ONLY
-    if (getenv("UPDATE")) {
-        mbgl::filesystem::create_directories(metadata.paths.platformExpectations);
-        mbgl::util::write_file(metadata.paths.platformExpectations.string() + "/expected.png", mbgl::encodePNG(actual));
+    if (getenv("UPDATE_PLATFORM")) {
+        mbgl::filesystem::create_directories(expectations.back());
+        mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actual));
         return true;
-    } else if (getenv("UPDATE_GENERIC")) {
+    } else if (getenv("UPDATE_DEFAULT")) {
         mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actual));
         return true;
     }
@@ -68,9 +71,19 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met
     mbgl::PremultipliedImage diff { actual.size };
 
     double pixels = 0.0;
-    const auto& expectedPath = mbgl::filesystem::exists(metadata.paths.platformExpectations) ?
-        metadata.paths.platformExpectations : metadata.paths.defaultExpectations;
+    mbgl::filesystem::path expectedPath;
+    for (auto rit = expectations.rbegin(); rit!= expectations.rend(); ++rit) {
+        if (mbgl::filesystem::exists(*rit)) {
+            expectedPath = *rit;
+            break;
+        }
+    }
 
+    if (expectedPath.empty()) {
+        metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string();
+        return false;
+    }
+    
     for (const auto& entry: readExpectedEntries(expectedPath)) {
         mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
         if (!maybeExpectedImage) {
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index 920a9ee372..74cc03ba03 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -17,7 +17,7 @@ public:
     /// Returns path of the render tests root directory.
     static const std::string& getBasePath();
     /// Returns path of mapbox-gl-native expectations directory.
-    static const std::string& getPlatformExpectationsPath();
+    static const std::vector<std::string>& getPlatformExpectationsPaths();
 
 private:
     bool runOperations(const std::string& key, TestMetadata&);