[render-test] Add metric_paths into Manifest (#15973)

* [render-test]Add expected metrics path in manifest file

* seperate metrics.json and expected.png

* fix expected path parsing

* keep expectation path if update is enabled

1 files changed, 31 insertions, 12 deletions

diff --git a/render-test/manifest_parser.cpp b/render-test/manifest_parser.cpp
index 6bba33754a..01d2246685 100644
--- a/render-test/manifest_parser.cpp
+++ b/render-test/manifest_parser.cpp
@@ -223,18 +223,16 @@ std::vector<std::pair<std::string, std::string>> parseIgnores(const std::vector<
     return ignores;
 }
 
-std::vector<mbgl::filesystem::path> getTestExpectations(mbgl::filesystem::path testPath,
-                                                        const mbgl::filesystem::path& testsRootPath,
-                                                        std::vector<mbgl::filesystem::path> expectationsPaths) {
-    std::vector<mbgl::filesystem::path> expectations{std::move(testPath.remove_filename())};
-    const auto& defaultTestExpectationsPath = expectations.front().string();
-
-    const std::regex regex{testsRootPath.string()};
-    for (const auto& path : expectationsPaths) {
-        expectations.emplace_back(std::regex_replace(defaultTestExpectationsPath, regex, path.string()));
-        assert(!expectations.back().empty());
+// defaultExpectationPath: absolute path that constains the style.json file for testing
+// testId: Test case id that used for composing expectation path
+// expectatedPaths: absolute paths that constain possible expected.png/metrics.json files for result checking
+std::vector<mbgl::filesystem::path> getTestExpectations(const mbgl::filesystem::path& defaultExpectationPath,
+                                                        const std::string& testId,
+                                                        std::vector<mbgl::filesystem::path> expectatedPaths) {
+    std::vector<mbgl::filesystem::path> expectations{defaultExpectationPath};
+    for (const auto& expectedPath : expectatedPaths) {
+        expectations.emplace_back(expectedPath / testId);
     }
-
     return expectations;
 }
 
@@ -331,6 +329,20 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manife
         }
         enbaleProbeTest = true;
     }
+    mbgl::filesystem::path expectedMetricPath;
+    if (document.HasMember("metric_path")) {
+        const auto& metricPathValue = document["metric_path"];
+        if (!metricPathValue.IsString()) {
+            mbgl::Log::Warning(mbgl::Event::General,
+                               "Invalid metric_path is provoided inside the manifest file: %s",
+                               filePath.c_str());
+            return mbgl::nullopt;
+        }
+        expectedMetricPath = getValidPath(manifest.manifestPath, metricPathValue.GetString());
+        if (expectedMetricPath.empty()) {
+            return mbgl::nullopt;
+        }
+    }
     std::vector<mbgl::filesystem::path> expectationPaths{};
     if (document.HasMember("expectation_paths")) {
         const auto& expectationPathValue = document["expectation_paths"];
@@ -414,7 +426,14 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manife
             }
 
             if (testPath.path().filename() == "style.json") {
-                testPaths.emplace_back(testPath, getTestExpectations(testPath, path, expectationPaths));
+                const auto defaultExpectationPath{std::move(mbgl::filesystem::path(testPath).remove_filename())};
+                const auto rootLength = manifest.testRootPath.length();
+                auto testId = defaultExpectationPath.string();
+                testId = testId.substr(rootLength + 1, testId.length() - rootLength - 1);
+
+                testPaths.emplace_back(testPath,
+                                       getTestExpectations(defaultExpectationPath, testId, expectationPaths),
+                                       getTestExpectations(defaultExpectationPath, testId, {expectedMetricPath}));
             }
         }
     }