From d7c8104decbbea8203bd79a95c0647bc4e1a2196 Mon Sep 17 00:00:00 2001 From: zmiao Date: Tue, 28 Jan 2020 21:48:22 +0200 Subject: [test-runner] Enable rebaseline for next-ios-render-test-runner (#16147) --- render-test/ios/README.md | 7 ++++++ render-test/ios/iosTestRunner.h | 1 + render-test/ios/iosTestRunner.mm | 51 +++++++++++++++++++++++++++++++++++++--- render-test/ios/tests/Tests.m | 14 ++++++++++- render-test/manifest_parser.cpp | 2 ++ 5 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 render-test/ios/README.md (limited to 'render-test') diff --git a/render-test/ios/README.md b/render-test/ios/README.md new file mode 100644 index 0000000000..d4c92bc7c3 --- /dev/null +++ b/render-test/ios/README.md @@ -0,0 +1,7 @@ +# iOS RenderTestRunner App + +This is a blank single-view-controller iOS app, linked against to a C++ static libraries, plus a a simple unit xctest. We use CMake to create an Xcode-friendly out-of-source build system. In another word, the build system is maintained by `CMakeLists` file,instead of a configured `.xcodeproj` file. + +This CMake project can build the executable RenderTestApp which is linked to the static C++ library 'mbgl-render-test'. CMakeLists.txt files are the only build configuration kept in source control. This is in contrast to committing the `.xcodeproj` directory which includes the backing XML, which is nonsensically hard to edit by hand. + +The test instantiates ObjC object of class `IosTestRunner` from the app, the class will instantiates a C++ object from the linked library `mbgl-render-test` and calls running render test function on it. It subsequently deletes the C++ object pointer. In the end, the test will check the existence of test reports, which are html files that attach to xctest result. diff --git a/render-test/ios/iosTestRunner.h b/render-test/ios/iosTestRunner.h index d2dd2e4ae8..7dc2e24ef5 100644 --- a/render-test/ios/iosTestRunner.h +++ b/render-test/ios/iosTestRunner.h @@ -5,6 +5,7 @@ __attribute__((visibility ("default"))) - (NSString*) getStyleResultPath; - (NSString*) getMetricResultPath; +- (NSString*) getMetricPath; - (BOOL) getTestStatus; @end diff --git a/render-test/ios/iosTestRunner.mm b/render-test/ios/iosTestRunner.mm index 3fe16c1d8e..4c3f7b87f1 100644 --- a/render-test/ios/iosTestRunner.mm +++ b/render-test/ios/iosTestRunner.mm @@ -2,6 +2,8 @@ #include "ios_test_runner.hpp" +#import "SSZipArchive.h" + #include @interface IosTestRunner () @@ -90,6 +92,45 @@ NSLog(@"Metric test result file '%@' doese not exit ", self.metricResultPath); self.testStatus = NO; } + + NSString *rebaselinePath = [path stringByAppendingPathComponent:@"/baselines"]; + BOOL needArchiving = NO; + BOOL isDir = NO; + NSArray *subpaths = [[NSArray alloc] init]; + if ([fileManager fileExistsAtPath:rebaselinePath isDirectory: &isDir] && isDir){ + subpaths = [fileManager subpathsAtPath:rebaselinePath]; + for(NSString *path in subpaths) + { + NSString *longPath = [rebaselinePath stringByAppendingPathComponent:path]; + if([fileManager fileExistsAtPath:longPath isDirectory:&isDir] && !isDir) + { + needArchiving = YES; + break; + } + } + } + else { + NSLog(@"Metric path '%@' doese not exit ", rebaselinePath); + } + + if (needArchiving) { + NSString *archivePath = [path stringByAppendingString:@"/metrics.zip"]; + BOOL success = [SSZipArchive createZipFileAtPath:archivePath + withContentsOfDirectory:rebaselinePath + keepParentDirectory:NO + compressionLevel:-1 + password:nil + AES:YES + progressHandler:nil]; + + if (success) { + NSLog(@"Successfully archive all of the metrics into metrics.zip"); + self.metricPath = archivePath; + } + else { + NSLog(@"Failed to archive rebaselined metrics into metrics.zip"); + } + } } delete self.runner; @@ -99,14 +140,18 @@ } - (NSString*) getStyleResultPath { - return self.styleResultPath; + return self.styleResultPath; } - (NSString*) getMetricResultPath { - return self.metricResultPath; + return self.metricResultPath; +} + +- (NSString*) getMetricPath { + return self.metricPath; } - (BOOL) getTestStatus { - return self.testStatus; + return self.testStatus; } @end diff --git a/render-test/ios/tests/Tests.m b/render-test/ios/tests/Tests.m index c8fef0fa8a..0b7f3c99dc 100644 --- a/render-test/ios/tests/Tests.m +++ b/render-test/ios/tests/Tests.m @@ -31,7 +31,7 @@ XCTAssert(attachment1URL, @"Failed to attach test result '%@'", styleResult); attachment1URL.lifetime = XCTAttachmentLifetimeKeepAlways; [self addAttachment:attachment1URL]; - + fileFound = [fileManager fileExistsAtPath: metricResult]; XCTAssert(fileFound, @"Test result html '%@' doese not exit", metricResult); NSURL *metricURL = [NSURL fileURLWithPath:metricResult]; @@ -40,6 +40,18 @@ attachment2URL.lifetime = XCTAttachmentLifetimeKeepAlways; [self addAttachment:attachment2URL]; + NSString* metrics = [runner getMetricPath]; + if (metrics && [fileManager fileExistsAtPath: metrics]) { + NSURL *metricsURL = [NSURL fileURLWithPath:metrics]; + XCTAttachment *attachmentMetricsURL = [XCTAttachment attachmentWithContentsOfFileAtURL: metricsURL]; + XCTAssert(attachmentMetricsURL, @"Failed to attach test rebaselined metrics '%@'", metrics); + attachmentMetricsURL.lifetime = XCTAttachmentLifetimeKeepAlways; + [self addAttachment:attachmentMetricsURL]; + } + else { + NSLog(@"No rebaselined metrics are found"); + } + BOOL success = [runner getTestStatus]; XCTAssert(success, @"IOSTestRunner reports error because some of the tests are not passed, please check the test report"); } diff --git a/render-test/manifest_parser.cpp b/render-test/manifest_parser.cpp index 4d2952c761..2e1b34df91 100644 --- a/render-test/manifest_parser.cpp +++ b/render-test/manifest_parser.cpp @@ -280,6 +280,8 @@ mbgl::optional ManifestParser::parseManifest(const std::string& manife std::vector expectedMetricPaths{expectedMetricPath}; #if defined(__ANDROID__) expectedMetricPaths.emplace_back("/sdcard/baselines/"); +#elif defined(__APPLE__) + expectedMetricPaths.emplace_back(manifest.manifestPath + "/baselines/"); #endif testPaths.emplace_back(testPath, getTestExpectations(defaultExpectationPath, testId, expectationPaths), -- cgit v1.2.1