From feb1fb9f36a7762f40d83fd350cafd95f6ba4651 Mon Sep 17 00:00:00 2001 From: zmiao Date: Thu, 28 Nov 2019 15:05:16 +0200 Subject: Make renderTestrunner run two times with different manifest config --- .gitignore | 1 + circle.yml | 2 +- next/platform/android/android.cmake | 12 +++++-- platform/android/src/test/render_test_runner.cpp | 39 +++++++++++++--------- render-test/android-manifest-probe-memory.json | 10 ++++++ .../android-manifest-probe-network-gfx.json | 10 ++++++ render-test/android/app/src/main/assets/to_zip.txt | 3 +- 7 files changed, 56 insertions(+), 21 deletions(-) create mode 100644 render-test/android-manifest-probe-memory.json create mode 100644 render-test/android-manifest-probe-network-gfx.json diff --git a/.gitignore b/.gitignore index 33d3d987d8..d81b8c9727 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ xcuserdata /platform/ios/benchmark/assets/glyphs/DIN* /platform/ios/benchmark/assets/tiles/mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6 /platform/darwin/developer.xcconfig +**/token /platform/macos/macos.xcworkspace/xcshareddata/macos.xcscmblueprint /platform/ios/ios.xcworkspace/xcshareddata/ios.xcscmblueprint /documentation diff --git a/circle.yml b/circle.yml index 447b98f0a4..27cdf7c0ea 100644 --- a/circle.yml +++ b/circle.yml @@ -707,7 +707,7 @@ jobs: --device-ids flame --os-version-ids 29 --locales en --orientations portrait --timeout 20m \ --directories-to-pull /sdcard --results-dir render-test-app-${CIRCLE_BUILD_NUM} \ --no-record-video --no-performance-metrics - testResult=$(gsutil ls gs://test-lab-186672a0qp5bq-ycr70axads3nc/render-test-app-${CIRCLE_BUILD_NUM}/**/*.html | tail -1) + testResult=$(gsutil ls gs://test-lab-186672a0qp5bq-ycr70axads3nc/render-test-app-${CIRCLE_BUILD_NUM}/**/*.html) mkdir -p /tmp/tests/render-test-app gsutil cp $testResult /tmp/tests/render-test-app | true fi diff --git a/next/platform/android/android.cmake b/next/platform/android/android.cmake index b0c9c712dc..befdc6f31a 100644 --- a/next/platform/android/android.cmake +++ b/next/platform/android/android.cmake @@ -344,8 +344,14 @@ add_custom_command( ${CMAKE_COMMAND} -E copy - ${MBGL_ROOT}/render-test/android-manifest.json - ${MBGL_ROOT}/android-manifest.json + ${MBGL_ROOT}/render-test/android-manifest-probe-memory.json + ${MBGL_ROOT}/android-manifest-probe-memory.json + COMMAND + ${CMAKE_COMMAND} + -E + copy + ${MBGL_ROOT}/render-test/android-manifest-probe-network-gfx.json + ${MBGL_ROOT}/android-manifest-probe-network-gfx.json COMMAND ${CMAKE_COMMAND} -E @@ -362,7 +368,7 @@ add_custom_command( ${CMAKE_COMMAND} -E tar - "cf" + "chvf" "render-test/android/app/src/main/assets/data.zip" --format=zip --files-from=render-test/android/app/src/main/assets/to_zip.txt diff --git a/platform/android/src/test/render_test_runner.cpp b/platform/android/src/test/render_test_runner.cpp index 98197ce1cc..cef89e9448 100644 --- a/platform/android/src/test/render_test_runner.cpp +++ b/platform/android/src/test/render_test_runner.cpp @@ -252,26 +252,33 @@ void android_main(struct android_app* app) { } else { unZipFile(env, zipFile, storagePath); - std::string configFile = storagePath + "/android-manifest.json"; - std::vector arguments = {"mbgl-render-test-runner", "-p", configFile}; - std::vector argv; - for (const auto& arg : arguments) { - argv.push_back((char*)arg.data()); - } - argv.push_back(nullptr); + auto runTestWithManifest = [&](const std::string manifest, const std::string filter = {}) -> bool { + std::string configFile = storagePath + manifest; + std::vector arguments = {"mbgl-render-test-runner", "-p", configFile, filter}; + std::vector argv; + for (const auto& arg : arguments) { + argv.push_back((char*)arg.data()); + } + argv.push_back(nullptr); - int finishedTestCount = 0; - std::function testStatus = [&]() { - ALooper_pollAll(0, &outFd, &outEvents, reinterpret_cast(&source)); + int finishedTestCount = 0; + std::function testStatus = [&]() { + ALooper_pollAll(0, &outFd, &outEvents, reinterpret_cast(&source)); - if (source != nullptr) { - source->process(app, source); - } + if (source != nullptr) { + source->process(app, source); + } - mbgl::Log::Info(mbgl::Event::General, "Current finished tests number is '%d' ", ++finishedTestCount); + mbgl::Log::Info(mbgl::Event::General, "Current finished tests number is '%d' ", ++finishedTestCount); + }; + mbgl::Log::Info( + mbgl::Event::General, "Start running RenderTestRunner with manifest: '%s'", manifest.c_str()); + bool result = mbgl::runRenderTests(argv.size() - 1, argv.data(), testStatus) == 0; + mbgl::Log::Info(mbgl::Event::General, "End running RenderTestRunner with manifest: '%s'", manifest.c_str()); + return result; }; - - mbgl::runRenderTests(argv.size() - 1, argv.data(), testStatus); + runTestWithManifest("/android-manifest-probe-network-gfx.json"); + runTestWithManifest("/android-manifest-probe-memory.json", "--filter=real-world"); mbgl::Log::Info(mbgl::Event::General, "All tests are finished!"); changeState(env, app); } diff --git a/render-test/android-manifest-probe-memory.json b/render-test/android-manifest-probe-memory.json new file mode 100644 index 0000000000..bc94465761 --- /dev/null +++ b/render-test/android-manifest-probe-memory.json @@ -0,0 +1,10 @@ +{ + "base_test_path":"mapbox-gl-js/test/integration", + "expectation_paths":["render-test/expected"], + "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"], + "vendor_path":"vendor", + "asset_path": "mapbox-gl-js/test/integration", + "result_path":"/sdcard/", + "probes": ["probeMemory"], + "metric_path":"render-test/metrics/android" +} \ No newline at end of file diff --git a/render-test/android-manifest-probe-network-gfx.json b/render-test/android-manifest-probe-network-gfx.json new file mode 100644 index 0000000000..b16424d999 --- /dev/null +++ b/render-test/android-manifest-probe-network-gfx.json @@ -0,0 +1,10 @@ +{ + "base_test_path":"mapbox-gl-js/test/integration", + "expectation_paths":["render-test/expected"], + "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"], + "vendor_path":"vendor", + "asset_path": "mapbox-gl-js/test/integration", + "result_path":"/sdcard/", + "probes": ["probeGFX", "probeNetwork"], + "metric_path":"render-test/metrics/android" +} \ No newline at end of file diff --git a/render-test/android/app/src/main/assets/to_zip.txt b/render-test/android/app/src/main/assets/to_zip.txt index aed98772c9..fbf9539583 100644 --- a/render-test/android/app/src/main/assets/to_zip.txt +++ b/render-test/android/app/src/main/assets/to_zip.txt @@ -15,4 +15,5 @@ vendor/mvt-fixtures/real-world/ ignores/ render-test/metrics/android/ render-test/expected/ -android-manifest.json \ No newline at end of file +android-manifest-probe-network-gfx.json +android-manifest-probe-memory.json \ No newline at end of file -- cgit v1.2.1