From d7c8104decbbea8203bd79a95c0647bc4e1a2196 Mon Sep 17 00:00:00 2001 From: zmiao Date: Tue, 28 Jan 2020 21:48:22 +0200 Subject: [test-runner] Enable rebaseline for next-ios-render-test-runner (#16147) --- .gitmodules | 3 ++ circle.yml | 7 +++++ next/platform/ios/ios.cmake | 5 ++++ render-test/ios/README.md | 7 +++++ render-test/ios/iosTestRunner.h | 1 + render-test/ios/iosTestRunner.mm | 51 +++++++++++++++++++++++++++++++-- render-test/ios/tests/Tests.m | 14 ++++++++- render-test/manifest_parser.cpp | 2 ++ vendor/zip-archive | 1 + vendor/zip-archive.cmake | 61 ++++++++++++++++++++++++++++++++++++++++ 10 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 render-test/ios/README.md create mode 160000 vendor/zip-archive create mode 100644 vendor/zip-archive.cmake diff --git a/.gitmodules b/.gitmodules index ff4c0bbed5..d84f3057b7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -70,3 +70,6 @@ [submodule "vendor/googletest"] path = vendor/googletest url = https://github.com/google/googletest.git +[submodule "vendor/zip-archive"] + path = vendor/zip-archive + url = https://github.com/ZipArchive/ZipArchive.git diff --git a/circle.yml b/circle.yml index 7b062bfd69..5427bce6ca 100644 --- a/circle.yml +++ b/circle.yml @@ -30,6 +30,7 @@ workflows: - linux-clang8-release - linux-gcc8-release - macos-xcode11-release + - ios-render-test-runner - build-template: name: android-armeabi-v7a-release executor_name: ubuntu-disco @@ -389,6 +390,12 @@ jobs: xcparse attachments *.xcresult && ls cd ../.. && mkdir -p /tmp/tests/ios-render-test if ls render-test-app/TestLogs/*.html 1> /dev/null 2>&1; then cp render-test-app/TestLogs/*.html /tmp/tests/ios-render-test; fi + if [[ -n $(ls render-test-app/TestLogs/*.zip 2>/dev/null) ]]; then + cp render-test-app/TestLogs/*.zip metrics/next-ios-render-test-runner/ + cd metrics/next-ios-render-test-runner/ + unzip *.zip + rm -rf *.zip + fi fi - save android-benchmark-runner: diff --git a/next/platform/ios/ios.cmake b/next/platform/ios/ios.cmake index 29cf4a5ca0..68578e0dfa 100644 --- a/next/platform/ios/ios.cmake +++ b/next/platform/ios/ios.cmake @@ -95,6 +95,10 @@ target_link_libraries( if(MBGL_IOS_RENDER_TEST) set(CMAKE_OSX_ARCHITECTURES "armv7;i386;x86_64;arm64") + + include(${PROJECT_SOURCE_DIR}/vendor/zip-archive.cmake) + initialize_ios_target(mbgl-vendor-zip-archive) + set(PREPARE_CMD "${MBGL_ROOT}/render-test/ios/setup_test_data.sh") message("COMMAND: ${PREPARE_CMD}") execute_process(COMMAND ${PREPARE_CMD} RESULT_VARIABLE CMD_ERROR) @@ -160,6 +164,7 @@ if(MBGL_IOS_RENDER_TEST) "-framework QuartzCore" "-framework UIKit" mbgl-render-test + mbgl-vendor-zip-archive ) find_package(XCTest REQUIRED) diff --git a/render-test/ios/README.md b/render-test/ios/README.md new file mode 100644 index 0000000000..d4c92bc7c3 --- /dev/null +++ b/render-test/ios/README.md @@ -0,0 +1,7 @@ +# iOS RenderTestRunner App + +This is a blank single-view-controller iOS app, linked against to a C++ static libraries, plus a a simple unit xctest. We use CMake to create an Xcode-friendly out-of-source build system. In another word, the build system is maintained by `CMakeLists` file,instead of a configured `.xcodeproj` file. + +This CMake project can build the executable RenderTestApp which is linked to the static C++ library 'mbgl-render-test'. CMakeLists.txt files are the only build configuration kept in source control. This is in contrast to committing the `.xcodeproj` directory which includes the backing XML, which is nonsensically hard to edit by hand. + +The test instantiates ObjC object of class `IosTestRunner` from the app, the class will instantiates a C++ object from the linked library `mbgl-render-test` and calls running render test function on it. It subsequently deletes the C++ object pointer. In the end, the test will check the existence of test reports, which are html files that attach to xctest result. diff --git a/render-test/ios/iosTestRunner.h b/render-test/ios/iosTestRunner.h index d2dd2e4ae8..7dc2e24ef5 100644 --- a/render-test/ios/iosTestRunner.h +++ b/render-test/ios/iosTestRunner.h @@ -5,6 +5,7 @@ __attribute__((visibility ("default"))) - (NSString*) getStyleResultPath; - (NSString*) getMetricResultPath; +- (NSString*) getMetricPath; - (BOOL) getTestStatus; @end diff --git a/render-test/ios/iosTestRunner.mm b/render-test/ios/iosTestRunner.mm index 3fe16c1d8e..4c3f7b87f1 100644 --- a/render-test/ios/iosTestRunner.mm +++ b/render-test/ios/iosTestRunner.mm @@ -2,6 +2,8 @@ #include "ios_test_runner.hpp" +#import "SSZipArchive.h" + #include @interface IosTestRunner () @@ -90,6 +92,45 @@ NSLog(@"Metric test result file '%@' doese not exit ", self.metricResultPath); self.testStatus = NO; } + + NSString *rebaselinePath = [path stringByAppendingPathComponent:@"/baselines"]; + BOOL needArchiving = NO; + BOOL isDir = NO; + NSArray *subpaths = [[NSArray alloc] init]; + if ([fileManager fileExistsAtPath:rebaselinePath isDirectory: &isDir] && isDir){ + subpaths = [fileManager subpathsAtPath:rebaselinePath]; + for(NSString *path in subpaths) + { + NSString *longPath = [rebaselinePath stringByAppendingPathComponent:path]; + if([fileManager fileExistsAtPath:longPath isDirectory:&isDir] && !isDir) + { + needArchiving = YES; + break; + } + } + } + else { + NSLog(@"Metric path '%@' doese not exit ", rebaselinePath); + } + + if (needArchiving) { + NSString *archivePath = [path stringByAppendingString:@"/metrics.zip"]; + BOOL success = [SSZipArchive createZipFileAtPath:archivePath + withContentsOfDirectory:rebaselinePath + keepParentDirectory:NO + compressionLevel:-1 + password:nil + AES:YES + progressHandler:nil]; + + if (success) { + NSLog(@"Successfully archive all of the metrics into metrics.zip"); + self.metricPath = archivePath; + } + else { + NSLog(@"Failed to archive rebaselined metrics into metrics.zip"); + } + } } delete self.runner; @@ -99,14 +140,18 @@ } - (NSString*) getStyleResultPath { - return self.styleResultPath; + return self.styleResultPath; } - (NSString*) getMetricResultPath { - return self.metricResultPath; + return self.metricResultPath; +} + +- (NSString*) getMetricPath { + return self.metricPath; } - (BOOL) getTestStatus { - return self.testStatus; + return self.testStatus; } @end diff --git a/render-test/ios/tests/Tests.m b/render-test/ios/tests/Tests.m index c8fef0fa8a..0b7f3c99dc 100644 --- a/render-test/ios/tests/Tests.m +++ b/render-test/ios/tests/Tests.m @@ -31,7 +31,7 @@ XCTAssert(attachment1URL, @"Failed to attach test result '%@'", styleResult); attachment1URL.lifetime = XCTAttachmentLifetimeKeepAlways; [self addAttachment:attachment1URL]; - + fileFound = [fileManager fileExistsAtPath: metricResult]; XCTAssert(fileFound, @"Test result html '%@' doese not exit", metricResult); NSURL *metricURL = [NSURL fileURLWithPath:metricResult]; @@ -40,6 +40,18 @@ attachment2URL.lifetime = XCTAttachmentLifetimeKeepAlways; [self addAttachment:attachment2URL]; + NSString* metrics = [runner getMetricPath]; + if (metrics && [fileManager fileExistsAtPath: metrics]) { + NSURL *metricsURL = [NSURL fileURLWithPath:metrics]; + XCTAttachment *attachmentMetricsURL = [XCTAttachment attachmentWithContentsOfFileAtURL: metricsURL]; + XCTAssert(attachmentMetricsURL, @"Failed to attach test rebaselined metrics '%@'", metrics); + attachmentMetricsURL.lifetime = XCTAttachmentLifetimeKeepAlways; + [self addAttachment:attachmentMetricsURL]; + } + else { + NSLog(@"No rebaselined metrics are found"); + } + BOOL success = [runner getTestStatus]; XCTAssert(success, @"IOSTestRunner reports error because some of the tests are not passed, please check the test report"); } diff --git a/render-test/manifest_parser.cpp b/render-test/manifest_parser.cpp index 4d2952c761..2e1b34df91 100644 --- a/render-test/manifest_parser.cpp +++ b/render-test/manifest_parser.cpp @@ -280,6 +280,8 @@ mbgl::optional ManifestParser::parseManifest(const std::string& manife std::vector expectedMetricPaths{expectedMetricPath}; #if defined(__ANDROID__) expectedMetricPaths.emplace_back("/sdcard/baselines/"); +#elif defined(__APPLE__) + expectedMetricPaths.emplace_back(manifest.manifestPath + "/baselines/"); #endif testPaths.emplace_back(testPath, getTestExpectations(defaultExpectationPath, testId, expectationPaths), diff --git a/vendor/zip-archive b/vendor/zip-archive new file mode 160000 index 0000000000..44970effad --- /dev/null +++ b/vendor/zip-archive @@ -0,0 +1 @@ +Subproject commit 44970effad727e97a28a0b21e1f5a298949cb29a diff --git a/vendor/zip-archive.cmake b/vendor/zip-archive.cmake new file mode 100644 index 0000000000..b22586b413 --- /dev/null +++ b/vendor/zip-archive.cmake @@ -0,0 +1,61 @@ +if(TARGET mbgl-vendor-zip-archive) + return() +endif() + +add_library( + mbgl-vendor-zip-archive STATIC + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/SSZipArchive.h + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/SSZipArchive.m + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/SSZipCommon.h + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/ZipArchive.h + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz.h + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_compat.c + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_compat.h + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_crypt.c + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_crypt.h + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_crypt_apple.c + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_os.c + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_os.h + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_os_posix.c + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm.c + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm.h + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_buf.c + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_buf.h + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_mem.c + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_mem.h + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_os.h + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_os_posix.c + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_pkcrypt.c + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_pkcrypt.h + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_split.c + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_split.h + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_wzaes.c + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_wzaes.h + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_zlib.c + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_zlib.h + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_zip.c + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_zip.h + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_zip_rw.c + ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_zip_rw.h +) + +target_link_libraries( + mbgl-vendor-zip-archive + PRIVATE z +) + +target_include_directories( + mbgl-vendor-zip-archive SYSTEM + INTERFACE ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive +) + +target_compile_definitions( + mbgl-vendor-zip-archive + PRIVATE + HAVE_INTTYPES_H + HAVE_PKCRYPT + HAVE_STDINT_H + HAVE_WZAES + HAVE_ZLIB +) +set_property(TARGET mbgl-vendor-zip-archive PROPERTY FOLDER RenderTestApp) -- cgit v1.2.1