From 429292fb9898cbcf3f2a0b644e01277473124252 Mon Sep 17 00:00:00 2001 From: zmiao Date: Fri, 24 Jan 2020 15:45:47 +0200 Subject: [test-runner] Add iOS RenderTestRunner test metrics (#16139) * [test-runner] Add iOS RenderTestRunner test metrics * [test-runner] Add ios probe metrics --- render-test/ios/iosTestRunner.h | 6 +++++- render-test/ios/iosTestRunner.mm | 42 ++++++++++++++++++++++++------------- render-test/ios/ios_test_runner.cpp | 27 ++++++++++++++++-------- render-test/ios/setup_test_data.sh | 4 ++++ render-test/ios/tests/Tests.m | 31 +++++++++++++++++---------- render-test/runner.cpp | 2 +- 6 files changed, 76 insertions(+), 36 deletions(-) (limited to 'render-test') diff --git a/render-test/ios/iosTestRunner.h b/render-test/ios/iosTestRunner.h index c83a3db4c1..d2dd2e4ae8 100644 --- a/render-test/ios/iosTestRunner.h +++ b/render-test/ios/iosTestRunner.h @@ -1,6 +1,10 @@ #import + __attribute__((visibility ("default"))) @interface IosTestRunner : NSObject -- (NSString*) getResultPath; + +- (NSString*) getStyleResultPath; +- (NSString*) getMetricResultPath; - (BOOL) getTestStatus; + @end diff --git a/render-test/ios/iosTestRunner.mm b/render-test/ios/iosTestRunner.mm index 580c62c94d..5c0365d367 100644 --- a/render-test/ios/iosTestRunner.mm +++ b/render-test/ios/iosTestRunner.mm @@ -8,7 +8,11 @@ @property (nullable) TestRunner* runner; -@property (copy, nullable) NSString *resultPath; +@property (copy, nullable) NSString *styleResultPath; + +@property (copy, nullable) NSString *metricResultPath; + +@property (copy, nullable) NSString *metricPath; @property BOOL testStatus; @@ -20,7 +24,7 @@ { self = [super init]; if (self) { - self.testStatus = false; + self.testStatus = NO; self.runner = new TestRunner(); NSString *path = nil; NSError *error; @@ -69,17 +73,23 @@ } } if (path) { - NSString *manifestPath = [path stringByAppendingPathComponent:@"/next-ios-render-test-runner-style.json"]; - std::string manifest = std::string([manifestPath UTF8String]); - - self.testStatus = self.runner->startTest(manifest); - self.resultPath = [path stringByAppendingPathComponent:@"/next-ios-render-test-runner-style.html"]; - - BOOL fileFound = [fileManager fileExistsAtPath: self.resultPath]; - if (!fileFound) { - NSLog(@"File doese not exit %@", self.resultPath); + + std::string basePath = std::string([path UTF8String]); + self.testStatus = self.runner->startTest(basePath) ? YES : NO; + self.styleResultPath = [path stringByAppendingPathComponent:@"/next-ios-render-test-runner-style.html"]; + self.metricResultPath = [path stringByAppendingPathComponent:@"/next-ios-render-test-runner-metrics.html"]; + + BOOL fileFound = [fileManager fileExistsAtPath: self.styleResultPath]; + if (fileFound == NO) { + NSLog(@"Style test result file '%@' doese not exit ", self.styleResultPath); + self.testStatus = NO; + } + + fileFound = [fileManager fileExistsAtPath: self.metricResultPath]; + if (fileFound == NO) { + NSLog(@"Metric test result file '%@' doese not exit ", self.metricResultPath); + self.testStatus = NO; } - self.testStatus &= fileFound; } delete self.runner; @@ -88,8 +98,12 @@ return self; } -- (NSString*) getResultPath { - return self.resultPath; +- (NSString*) getStyleResultPath { + return self.styleResultPath; +} + +- (NSString*) getMetricResultPath { + return self.metricResultPath; } - (BOOL) getTestStatus { diff --git a/render-test/ios/ios_test_runner.cpp b/render-test/ios/ios_test_runner.cpp index 842318c64a..ceb44440aa 100644 --- a/render-test/ios/ios_test_runner.cpp +++ b/render-test/ios/ios_test_runner.cpp @@ -9,9 +9,9 @@ #define EXPORT __attribute__((visibility("default"))) EXPORT -bool TestRunner::startTest(const std::string& manifest) { +bool TestRunner::startTest(const std::string& manifestBasePath) { auto runTestWithManifest = [](const std::string& manifest) -> bool { - std::vector arguments = {"mbgl-render-test-runner", "-p", manifest}; + std::vector arguments = {"mbgl-render-test-runner", "-p", manifest, "-u", "rebaseline"}; std::vector argv; for (const auto& arg : arguments) { argv.push_back(const_cast(arg.data())); @@ -24,17 +24,26 @@ bool TestRunner::startTest(const std::string& manifest) { }; mbgl::Log::Info(mbgl::Event::General, "Start running RenderTestRunner with manifest: '%s'", manifest.c_str()); - auto result = mbgl::runRenderTests(static_cast(argv.size() - 1), argv.data(), testStatus) == 0; - mbgl::Log::Info(mbgl::Event::General, "End running RenderTestRunner with manifest: '%s'", manifest.c_str()); - return result; + auto result = mbgl::runRenderTests(static_cast(argv.size() - 1), argv.data(), testStatus); + + mbgl::Log::Info(mbgl::Event::General, + "End running RenderTestRunner with manifest: '%s' with result value %d", + manifest.c_str(), + result); + return result == 0; }; - auto ret = false; + bool status = false; try { - ret = runTestWithManifest(manifest); + status = runTestWithManifest(manifestBasePath + "/next-ios-render-test-runner-style.json"); + status = runTestWithManifest(manifestBasePath + "/next-ios-render-test-runner-metrics.json") && status; } catch (...) { - mbgl::Log::Info(mbgl::Event::General, "testFailed"); + mbgl::Log::Info(mbgl::Event::General, "Failed with exception"); } + mbgl::Log::Info(mbgl::Event::General, "All tests are finished!"); - return ret; + if (!status) { + mbgl::Log::Info(mbgl::Event::General, "There are failing test cases"); + } + return status; } diff --git a/render-test/ios/setup_test_data.sh b/render-test/ios/setup_test_data.sh index 9a1dc83820..a7d56827ce 100755 --- a/render-test/ios/setup_test_data.sh +++ b/render-test/ios/setup_test_data.sh @@ -2,11 +2,15 @@ rm -rf ../../test-data mkdir ../../test-data mkdir ../../test-data/integration +mkdir ../../test-data/baselines cp -r ../../mapbox-gl-js/test/integration/render-tests ../../test-data/integration/ cp -r ../../mapbox-gl-js/test/integration/query-tests ../../test-data/integration/ cp -r ../../metrics/expectations ../../test-data/ cp -r ../../metrics/ignores ../../test-data/ +cp -r ../../metrics/tests ../../test-data/ +cp -r ../../metrics/next-ios-render-test-runner ../../test-data/ cp -r ../../metrics/next-ios-render-test-runner-style.json ../../test-data/ +cp -r ../../metrics/next-ios-render-test-runner-metrics.json ../../test-data/ cp -r ../../metrics/cache-metrics.db ../../test-data/ cp -r ../../metrics/cache-style.db ../../test-data/ \ No newline at end of file diff --git a/render-test/ios/tests/Tests.m b/render-test/ios/tests/Tests.m index 5dd52a7fa3..c8fef0fa8a 100644 --- a/render-test/ios/tests/Tests.m +++ b/render-test/ios/tests/Tests.m @@ -17,19 +17,28 @@ - (void)testStartRenderTestRunner { IosTestRunner* runner = [[IosTestRunner alloc] init]; XCTAssert(runner, @"IOSTestRunner is not initialized correctly"); - - NSString* result = [runner getResultPath]; - XCTAssert(result, @"IOSTestRunner does not produce a result file"); + NSString* styleResult = [runner getStyleResultPath]; + XCTAssert(styleResult, @"IOSTestRunner does not produce a style test result file"); + NSString* metricResult = [runner getMetricResultPath]; + XCTAssert(metricResult, @"IOSTestRunner does not produce a metric test result file"); + NSFileManager *fileManager = [NSFileManager defaultManager]; - BOOL fileFound = [fileManager fileExistsAtPath: result]; - XCTAssert(fileFound, @"Test result html '%@' doese not exit", result); - - NSURL *url = [NSURL fileURLWithPath:result]; - XCTAttachment *attachmentURL = [XCTAttachment attachmentWithContentsOfFileAtURL: url]; - XCTAssert(attachmentURL, @"Failed to attach test result '%@'", result); - attachmentURL.lifetime = XCTAttachmentLifetimeKeepAlways; - [self addAttachment:attachmentURL]; + BOOL fileFound = [fileManager fileExistsAtPath: styleResult]; + XCTAssert(fileFound, @"Test result html '%@' doese not exit", styleResult); + NSURL *styleURL = [NSURL fileURLWithPath:styleResult]; + XCTAttachment *attachment1URL = [XCTAttachment attachmentWithContentsOfFileAtURL: styleURL]; + XCTAssert(attachment1URL, @"Failed to attach test result '%@'", styleResult); + attachment1URL.lifetime = XCTAttachmentLifetimeKeepAlways; + [self addAttachment:attachment1URL]; + + fileFound = [fileManager fileExistsAtPath: metricResult]; + XCTAssert(fileFound, @"Test result html '%@' doese not exit", metricResult); + NSURL *metricURL = [NSURL fileURLWithPath:metricResult]; + XCTAttachment *attachment2URL = [XCTAttachment attachmentWithContentsOfFileAtURL: metricURL]; + XCTAssert(attachment2URL, @"Failed to attach test result '%@'", metricResult); + attachment2URL.lifetime = XCTAttachmentLifetimeKeepAlways; + [self addAttachment:attachment2URL]; BOOL success = [runner getTestStatus]; XCTAssert(success, @"IOSTestRunner reports error because some of the tests are not passed, please check the test report"); diff --git a/render-test/runner.cpp b/render-test/runner.cpp index b0fdcc386b..3f59b6df4c 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -246,7 +246,7 @@ void TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage, mbgl::optional maybeExpectedImage = mbgl::util::readFile(entry); if (!maybeExpectedImage) { metadata.errorMessage = "Failed to load expected image " + entry; - metadata.renderErrored = true; + metadata.renderErrored++; return; } -- cgit v1.2.1