[test-runner] Add iOS RenderTestRunner test metrics (#16139)

* [test-runner] Add iOS RenderTestRunner test metrics

* [test-runner] Add ios probe metrics

6 files changed, 76 insertions, 36 deletions

diff --git a/render-test/ios/iosTestRunner.h b/render-test/ios/iosTestRunner.h
index c83a3db4c1..d2dd2e4ae8 100644
--- a/render-test/ios/iosTestRunner.h
+++ b/render-test/ios/iosTestRunner.h
@@ -1,6 +1,10 @@
 #import <Foundation/Foundation.h>
+
 __attribute__((visibility ("default")))
 @interface IosTestRunner : NSObject
-- (NSString*) getResultPath;
+
+- (NSString*) getStyleResultPath;
+- (NSString*) getMetricResultPath;
 - (BOOL) getTestStatus;
+
 @end
diff --git a/render-test/ios/iosTestRunner.mm b/render-test/ios/iosTestRunner.mm
index 580c62c94d..5c0365d367 100644
--- a/render-test/ios/iosTestRunner.mm
+++ b/render-test/ios/iosTestRunner.mm
@@ -8,7 +8,11 @@
 
 @property (nullable) TestRunner* runner;
 
-@property (copy, nullable) NSString *resultPath;
+@property (copy, nullable) NSString *styleResultPath;
+
+@property (copy, nullable) NSString *metricResultPath;
+
+@property (copy, nullable) NSString *metricPath;
 
 @property BOOL testStatus;
 
@@ -20,7 +24,7 @@
 {
     self = [super init];
     if (self) {
-        self.testStatus = false;
+        self.testStatus = NO;
         self.runner = new TestRunner();
         NSString *path = nil;
         NSError *error;
@@ -69,17 +73,23 @@
             }
         }
         if (path) {
-            NSString *manifestPath = [path stringByAppendingPathComponent:@"/next-ios-render-test-runner-style.json"];
-            std::string manifest = std::string([manifestPath UTF8String]);
-            
-            self.testStatus = self.runner->startTest(manifest);
-            self.resultPath =  [path stringByAppendingPathComponent:@"/next-ios-render-test-runner-style.html"];
-            
-            BOOL fileFound = [fileManager fileExistsAtPath: self.resultPath];
-            if (!fileFound) {
-                NSLog(@"File doese not exit %@", self.resultPath);
+
+            std::string basePath = std::string([path UTF8String]);
+            self.testStatus = self.runner->startTest(basePath) ? YES : NO;
+            self.styleResultPath =  [path stringByAppendingPathComponent:@"/next-ios-render-test-runner-style.html"];
+            self.metricResultPath =  [path stringByAppendingPathComponent:@"/next-ios-render-test-runner-metrics.html"];
+
+            BOOL fileFound = [fileManager fileExistsAtPath: self.styleResultPath];
+            if (fileFound == NO) {
+                NSLog(@"Style test result file '%@' doese not exit ", self.styleResultPath);
+                self.testStatus = NO;
+            }
+
+            fileFound = [fileManager fileExistsAtPath: self.metricResultPath];
+            if (fileFound == NO) {
+                NSLog(@"Metric test result file '%@' doese not exit ", self.metricResultPath);
+                self.testStatus = NO;
             }
-            self.testStatus &= fileFound;
         }
 
         delete self.runner;
@@ -88,8 +98,12 @@
     return self;
 }
 
-- (NSString*) getResultPath {
-   return self.resultPath;
+- (NSString*) getStyleResultPath {
+   return self.styleResultPath;
+}
+
+- (NSString*) getMetricResultPath {
+   return self.metricResultPath;
 }
 
 - (BOOL) getTestStatus {
diff --git a/render-test/ios/ios_test_runner.cpp b/render-test/ios/ios_test_runner.cpp
index 842318c64a..ceb44440aa 100644
--- a/render-test/ios/ios_test_runner.cpp
+++ b/render-test/ios/ios_test_runner.cpp
@@ -9,9 +9,9 @@
 #define EXPORT __attribute__((visibility("default")))
 
 EXPORT
-bool TestRunner::startTest(const std::string& manifest) {
+bool TestRunner::startTest(const std::string& manifestBasePath) {
     auto runTestWithManifest = [](const std::string& manifest) -> bool {
-        std::vector<std::string> arguments = {"mbgl-render-test-runner", "-p", manifest};
+        std::vector<std::string> arguments = {"mbgl-render-test-runner", "-p", manifest, "-u", "rebaseline"};
         std::vector<char*> argv;
         for (const auto& arg : arguments) {
             argv.push_back(const_cast<char*>(arg.data()));
@@ -24,17 +24,26 @@ bool TestRunner::startTest(const std::string& manifest) {
         };
         mbgl::Log::Info(mbgl::Event::General, "Start running RenderTestRunner with manifest: '%s'", manifest.c_str());
 
-        auto result = mbgl::runRenderTests(static_cast<int>(argv.size() - 1), argv.data(), testStatus) == 0;
-        mbgl::Log::Info(mbgl::Event::General, "End running RenderTestRunner with manifest: '%s'", manifest.c_str());
-        return result;
+        auto result = mbgl::runRenderTests(static_cast<int>(argv.size() - 1), argv.data(), testStatus);
+
+        mbgl::Log::Info(mbgl::Event::General,
+                        "End running RenderTestRunner with manifest: '%s' with result value %d",
+                        manifest.c_str(),
+                        result);
+        return result == 0;
     };
 
-    auto ret = false;
+    bool status = false;
     try {
-        ret = runTestWithManifest(manifest);
+        status = runTestWithManifest(manifestBasePath + "/next-ios-render-test-runner-style.json");
+        status = runTestWithManifest(manifestBasePath + "/next-ios-render-test-runner-metrics.json") && status;
     } catch (...) {
-        mbgl::Log::Info(mbgl::Event::General, "testFailed");
+        mbgl::Log::Info(mbgl::Event::General, "Failed with exception");
     }
+
     mbgl::Log::Info(mbgl::Event::General, "All tests are finished!");
-    return ret;
+    if (!status) {
+        mbgl::Log::Info(mbgl::Event::General, "There are failing test cases");
+    }
+    return status;
 }
diff --git a/render-test/ios/setup_test_data.sh b/render-test/ios/setup_test_data.sh
index 9a1dc83820..a7d56827ce 100755
--- a/render-test/ios/setup_test_data.sh
+++ b/render-test/ios/setup_test_data.sh
@@ -2,11 +2,15 @@
 rm -rf ../../test-data
 mkdir ../../test-data
 mkdir ../../test-data/integration
+mkdir ../../test-data/baselines
 
 cp -r ../../mapbox-gl-js/test/integration/render-tests ../../test-data/integration/
 cp -r ../../mapbox-gl-js/test/integration/query-tests ../../test-data/integration/
 cp -r ../../metrics/expectations ../../test-data/
 cp -r ../../metrics/ignores ../../test-data/
+cp -r ../../metrics/tests ../../test-data/
+cp -r ../../metrics/next-ios-render-test-runner ../../test-data/
 cp -r ../../metrics/next-ios-render-test-runner-style.json ../../test-data/
+cp -r ../../metrics/next-ios-render-test-runner-metrics.json ../../test-data/
 cp -r ../../metrics/cache-metrics.db ../../test-data/
 cp -r ../../metrics/cache-style.db ../../test-data/
\ No newline at end of file
diff --git a/render-test/ios/tests/Tests.m b/render-test/ios/tests/Tests.m
index 5dd52a7fa3..c8fef0fa8a 100644
--- a/render-test/ios/tests/Tests.m
+++ b/render-test/ios/tests/Tests.m
@@ -17,19 +17,28 @@
 - (void)testStartRenderTestRunner {
     IosTestRunner* runner = [[IosTestRunner alloc] init]; 
     XCTAssert(runner, @"IOSTestRunner is not initialized correctly");
-    
-    NSString* result = [runner getResultPath];
-    XCTAssert(result, @"IOSTestRunner does not produce a result file");
 
+    NSString* styleResult = [runner getStyleResultPath];
+    XCTAssert(styleResult, @"IOSTestRunner does not produce a style test result file");
+    NSString* metricResult = [runner getMetricResultPath];
+    XCTAssert(metricResult, @"IOSTestRunner does not produce a metric test result file");
+    
     NSFileManager *fileManager = [NSFileManager defaultManager];
-    BOOL fileFound = [fileManager fileExistsAtPath: result];
-    XCTAssert(fileFound, @"Test result html '%@' doese not exit", result);
-
-    NSURL *url = [NSURL fileURLWithPath:result]; 
-    XCTAttachment *attachmentURL = [XCTAttachment attachmentWithContentsOfFileAtURL: url];
-    XCTAssert(attachmentURL, @"Failed to attach test result '%@'", result);
-    attachmentURL.lifetime = XCTAttachmentLifetimeKeepAlways;
-    [self addAttachment:attachmentURL];
+    BOOL fileFound = [fileManager fileExistsAtPath: styleResult];
+    XCTAssert(fileFound, @"Test result html '%@' doese not exit", styleResult);
+    NSURL *styleURL = [NSURL fileURLWithPath:styleResult];
+    XCTAttachment *attachment1URL = [XCTAttachment attachmentWithContentsOfFileAtURL: styleURL];
+    XCTAssert(attachment1URL, @"Failed to attach test result '%@'", styleResult);
+    attachment1URL.lifetime = XCTAttachmentLifetimeKeepAlways;
+    [self addAttachment:attachment1URL];
+    
+    fileFound = [fileManager fileExistsAtPath: metricResult];
+    XCTAssert(fileFound, @"Test result html '%@' doese not exit", metricResult);
+    NSURL *metricURL = [NSURL fileURLWithPath:metricResult];
+    XCTAttachment *attachment2URL = [XCTAttachment attachmentWithContentsOfFileAtURL: metricURL];
+    XCTAssert(attachment2URL, @"Failed to attach test result '%@'", metricResult);
+    attachment2URL.lifetime = XCTAttachmentLifetimeKeepAlways;
+    [self addAttachment:attachment2URL];
 
     BOOL success = [runner getTestStatus];
     XCTAssert(success, @"IOSTestRunner reports error because some of the tests are not passed, please check the test report");
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index b0fdcc386b..3f59b6df4c 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -246,7 +246,7 @@ void TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage,
             mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
             if (!maybeExpectedImage) {
                 metadata.errorMessage = "Failed to load expected image " + entry;
-                metadata.renderErrored = true;
+                metadata.renderErrored++;
                 return;
             }