Make RenderTestRunner report test status

5 files changed, 22 insertions, 12 deletions

diff --git a/render-test/ios/iosTestRunner.h b/render-test/ios/iosTestRunner.h
index c68d3981d6..c83a3db4c1 100644
--- a/render-test/ios/iosTestRunner.h
+++ b/render-test/ios/iosTestRunner.h
@@ -2,4 +2,5 @@
 __attribute__((visibility ("default")))
 @interface IosTestRunner : NSObject
 - (NSString*) getResultPath;
+- (BOOL) getTestStatus;
 @end
diff --git a/render-test/ios/iosTestRunner.mm b/render-test/ios/iosTestRunner.mm
index 50531c0870..580c62c94d 100644
--- a/render-test/ios/iosTestRunner.mm
+++ b/render-test/ios/iosTestRunner.mm
@@ -10,6 +10,8 @@
 
 @property (copy, nullable) NSString *resultPath;
 
+@property BOOL testStatus;
+
 @end
 
 @implementation IosTestRunner
@@ -18,6 +20,7 @@
 {
     self = [super init];
     if (self) {
+        self.testStatus = false;
         self.runner = new TestRunner();
         NSString *path = nil;
         NSError *error;
@@ -53,7 +56,6 @@
                     if (!success){
                         NSAssert1(0, @"Failed to copy file '%@'.", [error localizedDescription]);
                         NSLog(@"Failed to copy %@ file, error %@", dirName, [error localizedDescription]);
-                    
                     }
                     else {
                         path = destinationPath;
@@ -68,17 +70,16 @@
         }
         if (path) {
             NSString *manifestPath = [path stringByAppendingPathComponent:@"/next-ios-render-test-runner-style.json"];
-            
             std::string manifest = std::string([manifestPath UTF8String]);
             
-            self.runner->startTest(manifest);
-            
+            self.testStatus = self.runner->startTest(manifest);
             self.resultPath =  [path stringByAppendingPathComponent:@"/next-ios-render-test-runner-style.html"];
             
-            BOOL success = [fileManager fileExistsAtPath: self.resultPath];
-            if (!success) {
+            BOOL fileFound = [fileManager fileExistsAtPath: self.resultPath];
+            if (!fileFound) {
                 NSLog(@"File doese not exit %@", self.resultPath);
             }
+            self.testStatus &= fileFound;
         }
 
         delete self.runner;
@@ -88,7 +89,10 @@
 }
 
 - (NSString*) getResultPath {
- 
    return self.resultPath;
 }
+
+- (BOOL) getTestStatus {
+   return self.testStatus;
+}
 @end
diff --git a/render-test/ios/ios_test_runner.cpp b/render-test/ios/ios_test_runner.cpp
index e612e549bc..842318c64a 100644
--- a/render-test/ios/ios_test_runner.cpp
+++ b/render-test/ios/ios_test_runner.cpp
@@ -9,7 +9,7 @@
 #define EXPORT __attribute__((visibility("default")))
 
 EXPORT
-void TestRunner::startTest(const std::string& manifest) {
+bool TestRunner::startTest(const std::string& manifest) {
     auto runTestWithManifest = [](const std::string& manifest) -> bool {
         std::vector<std::string> arguments = {"mbgl-render-test-runner", "-p", manifest};
         std::vector<char*> argv;
@@ -29,10 +29,12 @@ void TestRunner::startTest(const std::string& manifest) {
         return result;
     };
 
+    auto ret = false;
     try {
-        runTestWithManifest(manifest);
+        ret = runTestWithManifest(manifest);
     } catch (...) {
         mbgl::Log::Info(mbgl::Event::General, "testFailed");
     }
     mbgl::Log::Info(mbgl::Event::General, "All tests are finished!");
+    return ret;
 }
diff --git a/render-test/ios/ios_test_runner.hpp b/render-test/ios/ios_test_runner.hpp
index f6e9f59d77..4ec1da93b0 100644
--- a/render-test/ios/ios_test_runner.hpp
+++ b/render-test/ios/ios_test_runner.hpp
@@ -8,7 +8,7 @@ public:
     TestRunner() = default;
     ~TestRunner() = default;
 
-    void startTest(const std::string& manifest);
+    bool startTest(const std::string& manifest);
 };
 
 #endif /* ios_test_runner_hpp */
diff --git a/render-test/ios/tests/Tests.m b/render-test/ios/tests/Tests.m
index 7230c5d16c..5dd52a7fa3 100644
--- a/render-test/ios/tests/Tests.m
+++ b/render-test/ios/tests/Tests.m
@@ -22,14 +22,17 @@
     XCTAssert(result, @"IOSTestRunner does not produce a result file");
 
     NSFileManager *fileManager = [NSFileManager defaultManager];
-    BOOL success = [fileManager fileExistsAtPath: result];
-    XCTAssert(success, @"Test result html '%@' doese not exit", result);
+    BOOL fileFound = [fileManager fileExistsAtPath: result];
+    XCTAssert(fileFound, @"Test result html '%@' doese not exit", result);
 
     NSURL *url = [NSURL fileURLWithPath:result]; 
     XCTAttachment *attachmentURL = [XCTAttachment attachmentWithContentsOfFileAtURL: url];
     XCTAssert(attachmentURL, @"Failed to attach test result '%@'", result);
     attachmentURL.lifetime = XCTAttachmentLifetimeKeepAlways;
     [self addAttachment:attachmentURL];
+
+    BOOL success = [runner getTestStatus];
+    XCTAssert(success, @"IOSTestRunner reports error because some of the tests are not passed, please check the test report");
 }
 
 @end