diff options
31 files changed, 357 insertions, 31 deletions
diff --git a/circle.yml b/circle.yml index 1290338091..2cd1aa4b37 100644 --- a/circle.yml +++ b/circle.yml @@ -665,6 +665,12 @@ jobs: --device-ids flame --os-version-ids 29 --locales en --orientations portrait --timeout 20m \ --directories-to-pull /sdcard --results-dir render-test-app-${CIRCLE_BUILD_NUM} \ --no-record-video --no-performance-metrics + fi + - run: + name: Retrieve TestResult from gcloud + when: always + command: | + if [[ -n "${GCLOUD_SERVICE_ACCOUNT_JSON}" && -z "${SKIP_FIREBASE:-}" ]]; then testResult=$(gsutil ls gs://test-lab-186672a0qp5bq-ycr70axads3nc/render-test-app-${CIRCLE_BUILD_NUM}/**/*.html) mkdir -p /tmp/tests/render-test-app gsutil cp $testResult /tmp/tests/render-test-app | true diff --git a/next/platform/android/android.cmake b/next/platform/android/android.cmake index 0f8dd04315..29bc9c1326 100644 --- a/next/platform/android/android.cmake +++ b/next/platform/android/android.cmake @@ -357,29 +357,18 @@ add_custom_command( -E copy ${MBGL_ROOT}/platform/node/test/ignores.json - ${MBGL_ROOT}/ignores/ignores.json - COMMAND - ${CMAKE_COMMAND} - -E - copy - ${MBGL_ROOT}/render-test/linux-ignores.json - ${MBGL_ROOT}/ignores/linux-ignores.json + ${MBGL_ROOT}/render-test/ignores/ignores.json COMMAND ${CMAKE_COMMAND} -E tar - "chvf" + "chf" "render-test/android/app/src/main/assets/data.zip" --format=zip --files-from=render-test/android/app/src/main/assets/to_zip.txt COMMAND ${CMAKE_COMMAND} -E - remove_directory - ${MBGL_ROOT}/ignores - COMMAND - ${CMAKE_COMMAND} - -E remove ${MBGL_ROOT}/android-manifest* WORKING_DIRECTORY ${MBGL_ROOT} diff --git a/platform/android/src/test/render_test_runner.cpp b/platform/android/src/test/render_test_runner.cpp index d0af8d21a6..7e209aff3a 100644 --- a/platform/android/src/test/render_test_runner.cpp +++ b/platform/android/src/test/render_test_runner.cpp @@ -79,7 +79,7 @@ std::string jstringToStdString(JNIEnv* env, jstring jStr) { return ret; } -void changeState(JNIEnv* env, struct android_app* app) { +void changeState(JNIEnv* env, struct android_app* app, bool result) { jobject nativeActivity = app->activity->clazz; jclass acl = env->GetObjectClass(nativeActivity); jmethodID getClassLoader = env->GetMethodID(acl, "getClassLoader", "()Ljava/lang/ClassLoader;"); @@ -89,6 +89,8 @@ void changeState(JNIEnv* env, struct android_app* app) { JavaWrapper<jstring> strClassName(env, env->NewStringUTF("android.app.TestState")); jclass testStateClass = static_cast<jclass>(env->CallObjectMethod(cls, findClass, strClassName.get())); if (testStateClass != NULL) { + jfieldID id0 = env->GetStaticFieldID(testStateClass, "testResult", "Z"); + env->SetStaticBooleanField(testStateClass, id0, result); jfieldID id = env->GetStaticFieldID(testStateClass, "running", "Z"); env->SetStaticBooleanField(testStateClass, id, false); } @@ -278,10 +280,11 @@ void android_main(struct android_app* app) { mbgl::Log::Info(mbgl::Event::General, "End running RenderTestRunner with manifest: '%s'", manifest.c_str()); return result; }; - runTestWithManifest("/android-manifest-probe-network-gfx.json"); - runTestWithManifest("/android-manifest-probe-memory.json"); + + auto result = runTestWithManifest("/android-manifest-probe-network-gfx.json"); + result = runTestWithManifest("/android-manifest-probe-memory.json") && result; mbgl::Log::Info(mbgl::Event::General, "All tests are finished!"); - changeState(env, app); + changeState(env, app, result); } while (true) { ALooper_pollAll(0, &outFd, &outEvents, reinterpret_cast<void**>(&source)); diff --git a/render-test/android-manifest-probe-memory.json b/render-test/android-manifest-probe-memory.json index a3562088f4..a90d39ee83 100644 --- a/render-test/android-manifest-probe-memory.json +++ b/render-test/android-manifest-probe-memory.json @@ -1,7 +1,7 @@ { "base_test_path":"mapbox-gl-js/test/integration", - "expectation_paths":["render-test/expected"], - "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"], + "expectation_paths":["render-test/expected", "render-test/expected/android"], + "ignore_paths":["render-test/ignores/ignores.json", "render-test/ignores/linux-ignores.json", "render-test/ignores/android-temp-fail.json"], "vendor_path":"vendor", "asset_path": "mapbox-gl-js/test/integration", "result_path":"/sdcard/", diff --git a/render-test/android-manifest-probe-network-gfx.json b/render-test/android-manifest-probe-network-gfx.json index b16424d999..c7b45417d0 100644 --- a/render-test/android-manifest-probe-network-gfx.json +++ b/render-test/android-manifest-probe-network-gfx.json @@ -1,7 +1,7 @@ { "base_test_path":"mapbox-gl-js/test/integration", - "expectation_paths":["render-test/expected"], - "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"], + "expectation_paths":["render-test/expected", "render-test/expected/android"], + "ignore_paths":["render-test/ignores/ignores.json", "render-test/ignores/linux-ignores.json", "render-test/ignores/android-temp-fail.json"], "vendor_path":"vendor", "asset_path": "mapbox-gl-js/test/integration", "result_path":"/sdcard/", diff --git a/render-test/android-manifest.json b/render-test/android-manifest.json index ddc91d1933..00783aba60 100644 --- a/render-test/android-manifest.json +++ b/render-test/android-manifest.json @@ -1,10 +1,10 @@ { "base_test_path":"mapbox-gl-js/test/integration", - "expectation_paths":["render-test/expected"], + "expectation_paths":["render-test/expected", "render-test/expected/android"], "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"], "vendor_path":"vendor", "asset_path": "mapbox-gl-js/test/integration", - "result_path":"/sdcard/", + "result_path": "/sdcard/", "probes": ["probeGFX", "probeNetwork"], "metric_path":"render-test/metrics/android" -}
\ No newline at end of file +} diff --git a/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java b/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java index 3d8c333902..d508a9d5a4 100644 --- a/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java +++ b/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java @@ -6,6 +6,7 @@ import androidx.test.rule.ActivityTestRule; import org.junit.Rule; import org.junit.Test; +import org.junit.Assert; import org.junit.runner.RunWith; import android.util.Log; @@ -24,6 +25,8 @@ public class NativeActivityTest { Log.v("Test", "Test is running"); Thread.sleep(1000L); } - Log.v("Test", "End the test"); + Log.v("Test", "All render tests are finished!"); + Assert.assertTrue("All test cases are passed", TestState.testResult); + } }
\ No newline at end of file diff --git a/render-test/android/app/src/androidTest/java/android/app/TestState.java b/render-test/android/app/src/androidTest/java/android/app/TestState.java index 44a0653fb6..b374a72afe 100644 --- a/render-test/android/app/src/androidTest/java/android/app/TestState.java +++ b/render-test/android/app/src/androidTest/java/android/app/TestState.java @@ -2,4 +2,5 @@ package android.app; public class TestState { static boolean running = true; + static boolean testResult = true; }
\ No newline at end of file diff --git a/render-test/android/app/src/main/assets/to_zip.txt b/render-test/android/app/src/main/assets/to_zip.txt index fbf9539583..e440a7c514 100644 --- a/render-test/android/app/src/main/assets/to_zip.txt +++ b/render-test/android/app/src/main/assets/to_zip.txt @@ -12,8 +12,8 @@ mapbox-gl-js/test/integration/sprites/ vendor/mapbox-gl-styles/styles/ vendor/mapbox-gl-styles/sprites/ vendor/mvt-fixtures/real-world/ -ignores/ +render-test/ignores/ render-test/metrics/android/ render-test/expected/ android-manifest-probe-network-gfx.json -android-manifest-probe-memory.json
\ No newline at end of file +android-manifest-probe-memory.json diff --git a/render-test/expected/android/render-tests/combinations/symbol-translucent--fill-translucent/expected.png b/render-test/expected/android/render-tests/combinations/symbol-translucent--fill-translucent/expected.png Binary files differnew file mode 100644 index 0000000000..6a14209d0d --- /dev/null +++ b/render-test/expected/android/render-tests/combinations/symbol-translucent--fill-translucent/expected.png diff --git a/render-test/expected/android/render-tests/regressions/mapbox-gl-js#2305/expected.png b/render-test/expected/android/render-tests/regressions/mapbox-gl-js#2305/expected.png Binary files differnew file mode 100644 index 0000000000..99ca724f5b --- /dev/null +++ b/render-test/expected/android/render-tests/regressions/mapbox-gl-js#2305/expected.png diff --git a/render-test/expected/android/render-tests/regressions/mapbox-gl-js#5978/expected.png b/render-test/expected/android/render-tests/regressions/mapbox-gl-js#5978/expected.png Binary files differnew file mode 100644 index 0000000000..37fc500689 --- /dev/null +++ b/render-test/expected/android/render-tests/regressions/mapbox-gl-js#5978/expected.png diff --git a/render-test/expected/android/render-tests/sprites/1x-screen-1x-pattern/expected.png b/render-test/expected/android/render-tests/sprites/1x-screen-1x-pattern/expected.png Binary files differnew file mode 100644 index 0000000000..e4564aeee8 --- /dev/null +++ b/render-test/expected/android/render-tests/sprites/1x-screen-1x-pattern/expected.png diff --git a/render-test/expected/android/render-tests/sprites/1x-screen-2x-pattern/expected.png b/render-test/expected/android/render-tests/sprites/1x-screen-2x-pattern/expected.png Binary files differnew file mode 100644 index 0000000000..8cf3bb4bda --- /dev/null +++ b/render-test/expected/android/render-tests/sprites/1x-screen-2x-pattern/expected.png diff --git a/render-test/expected/android/render-tests/sprites/2x-screen-1x-pattern/expected.png b/render-test/expected/android/render-tests/sprites/2x-screen-1x-pattern/expected.png Binary files differnew file mode 100644 index 0000000000..0756d83652 --- /dev/null +++ b/render-test/expected/android/render-tests/sprites/2x-screen-1x-pattern/expected.png diff --git a/render-test/expected/android/render-tests/sprites/2x-screen-2x-pattern/expected.png b/render-test/expected/android/render-tests/sprites/2x-screen-2x-pattern/expected.png Binary files differnew file mode 100644 index 0000000000..a3b27f24a2 --- /dev/null +++ b/render-test/expected/android/render-tests/sprites/2x-screen-2x-pattern/expected.png diff --git a/render-test/ignores/android-temp-fail.json b/render-test/ignores/android-temp-fail.json new file mode 100644 index 0000000000..655c697528 --- /dev/null +++ b/render-test/ignores/android-temp-fail.json @@ -0,0 +1,9 @@ +{ + "render-tests/icon-text-fit/both-text-anchor-1x-image-2x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.", + "render-tests/icon-text-fit/both-text-anchor-2x-image-1x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.", + "render-tests/regressions/mapbox-gl-native#9976": "Needs to be investigated and fixed.", + "render-tests/regressions/mapbox-gl-js#5911a": "Needs to be investigated and fixed.", + "render-tests/line-pattern/overscaled": "Needs to be investigated and fixed.", + "render-tests/runtime-styling/image-add-pattern": "Needs to be investigated and fixed.", + "render-tests/runtime-styling/image-update-pattern": "Needs to be investigated and fixed." +} diff --git a/render-test/linux-ignores.json b/render-test/ignores/linux-ignores.json index d5a1a85c6c..d5a1a85c6c 100644 --- a/render-test/linux-ignores.json +++ b/render-test/ignores/linux-ignores.json diff --git a/render-test/linux-manifest-probe-gfx-network.json b/render-test/linux-manifest-probe-gfx-network.json index b63013ca18..fedf9fa6bd 100644 --- a/render-test/linux-manifest-probe-gfx-network.json +++ b/render-test/linux-manifest-probe-gfx-network.json @@ -1,7 +1,7 @@ { "base_test_path":"../mapbox-gl-js/test/integration", "expectation_paths":["expected/"], - "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"], + "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"], "vendor_path":"../vendor", "asset_path": "../mapbox-gl-js/test/integration", "probes": ["probeGFX", "probeNetwork"], diff --git a/render-test/linux-manifest-probe-memory.json b/render-test/linux-manifest-probe-memory.json index 25b59e4a44..e21326875d 100644 --- a/render-test/linux-manifest-probe-memory.json +++ b/render-test/linux-manifest-probe-memory.json @@ -1,7 +1,7 @@ { "base_test_path":"../mapbox-gl-js/test/integration", "expectation_paths":["expected/"], - "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"], + "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"], "vendor_path":"../vendor", "asset_path": "../mapbox-gl-js/test/integration", "probes": ["probeMemory"], diff --git a/render-test/linux-manifest.json b/render-test/linux-manifest.json index cc72a7446c..91b062c2cc 100644 --- a/render-test/linux-manifest.json +++ b/render-test/linux-manifest.json @@ -1,7 +1,7 @@ { "base_test_path":"../mapbox-gl-js/test/integration", "expectation_paths":["expected/"], - "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"], + "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"], "vendor_path":"../vendor", "asset_path": "../mapbox-gl-js/test/integration" }
\ No newline at end of file diff --git a/render-test/linux-query-manifest.json b/render-test/linux-query-manifest.json index a6723f5519..6d2071b19f 100644 --- a/render-test/linux-query-manifest.json +++ b/render-test/linux-query-manifest.json @@ -1,6 +1,6 @@ { "base_test_path":"../mapbox-gl-js/test/integration", - "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"], + "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"], "vendor_path":"../vendor", "asset_path": "../mapbox-gl-js/test/integration" }
\ No newline at end of file diff --git a/render-test/metrics/android/render-tests/line-dasharray/round/segments/metrics.json b/render-test/metrics/android/render-tests/line-dasharray/round/segments/metrics.json new file mode 100644 index 0000000000..d3af311705 --- /dev/null +++ b/render-test/metrics/android/render-tests/line-dasharray/round/segments/metrics.json @@ -0,0 +1,35 @@ +{ + "network": [ + [ + "probeNetwork - default - end", + 0, + 0 + ], + [ + "probeNetwork - default - start", + 0, + 0 + ] + ], + "gfx": [ + [ + "probeGFX - default - end", + 2, + 4, + 9, + 1, + [ + 36864, + 36864 + ], + [ + 202, + 202 + ], + [ + 704, + 704 + ] + ] + ] +}
\ No newline at end of file diff --git a/render-test/metrics/android/render-tests/line-dasharray/round/zero-gap-width/metrics.json b/render-test/metrics/android/render-tests/line-dasharray/round/zero-gap-width/metrics.json new file mode 100644 index 0000000000..d3af311705 --- /dev/null +++ b/render-test/metrics/android/render-tests/line-dasharray/round/zero-gap-width/metrics.json @@ -0,0 +1,35 @@ +{ + "network": [ + [ + "probeNetwork - default - end", + 0, + 0 + ], + [ + "probeNetwork - default - start", + 0, + 0 + ] + ], + "gfx": [ + [ + "probeGFX - default - end", + 2, + 4, + 9, + 1, + [ + 36864, + 36864 + ], + [ + 202, + 202 + ], + [ + 704, + 704 + ] + ] + ] +}
\ No newline at end of file diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-constant-size/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-constant-size/metrics.json new file mode 100644 index 0000000000..1ebdc9b090 --- /dev/null +++ b/render-test/metrics/android/render-tests/text-field/formatted-images-constant-size/metrics.json @@ -0,0 +1,35 @@ +{ + "network": [ + [ + "probeNetwork - default - end", + 3, + 161976 + ], + [ + "probeNetwork - default - start", + 0, + 0 + ] + ], + "gfx": [ + [ + "probeGFX - default - end", + 2, + 4, + 13, + 1, + [ + 24580, + 24580 + ], + [ + 190, + 190 + ], + [ + 1856, + 1856 + ] + ] + ] +}
\ No newline at end of file diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-line/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-line/metrics.json new file mode 100644 index 0000000000..f7f0edc2f0 --- /dev/null +++ b/render-test/metrics/android/render-tests/text-field/formatted-images-line/metrics.json @@ -0,0 +1,35 @@ +{ + "network": [ + [ + "probeNetwork - default - end", + 3, + 120865 + ], + [ + "probeNetwork - default - start", + 0, + 0 + ] + ], + "gfx": [ + [ + "probeGFX - default - end", + 3, + 4, + 11, + 1, + [ + 28318, + 28318 + ], + [ + 190, + 190 + ], + [ + 1424, + 1424 + ] + ] + ] +}
\ No newline at end of file diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-multiline/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-multiline/metrics.json new file mode 100644 index 0000000000..5dfadecdc9 --- /dev/null +++ b/render-test/metrics/android/render-tests/text-field/formatted-images-multiline/metrics.json @@ -0,0 +1,35 @@ +{ + "network": [ + [ + "probeNetwork - default - end", + 3, + 120865 + ], + [ + "probeNetwork - default - start", + 0, + 0 + ] + ], + "gfx": [ + [ + "probeGFX - default - end", + 2, + 6, + 13, + 1, + [ + 44921, + 44921 + ], + [ + 310, + 310 + ], + [ + 3136, + 3136 + ] + ] + ] +}
\ No newline at end of file diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-variable-anchors-justification/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-variable-anchors-justification/metrics.json new file mode 100644 index 0000000000..312dedfbf0 --- /dev/null +++ b/render-test/metrics/android/render-tests/text-field/formatted-images-variable-anchors-justification/metrics.json @@ -0,0 +1,35 @@ +{ + "network": [ + [ + "probeNetwork - default - end", + 3, + 161976 + ], + [ + "probeNetwork - default - start", + 0, + 0 + ] + ], + "gfx": [ + [ + "probeGFX - default - end", + 1, + 4, + 9, + 1, + [ + 24580, + 24580 + ], + [ + 2290, + 2290 + ], + [ + 24256, + 24256 + ] + ] + ] +}
\ No newline at end of file diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-vertical/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-vertical/metrics.json new file mode 100644 index 0000000000..e4a8816967 --- /dev/null +++ b/render-test/metrics/android/render-tests/text-field/formatted-images-vertical/metrics.json @@ -0,0 +1,35 @@ +{ + "network": [ + [ + "probeNetwork - default - end", + 4, + 255564 + ], + [ + "probeNetwork - default - start", + 0, + 0 + ] + ], + "gfx": [ + [ + "probeGFX - default - end", + 1, + 4, + 9, + 1, + [ + 35756, + 35756 + ], + [ + 310, + 310 + ], + [ + 3136, + 3136 + ] + ] + ] +}
\ No newline at end of file diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-zoom-dependent-size/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-zoom-dependent-size/metrics.json new file mode 100644 index 0000000000..7cf8a26c5d --- /dev/null +++ b/render-test/metrics/android/render-tests/text-field/formatted-images-zoom-dependent-size/metrics.json @@ -0,0 +1,35 @@ +{ + "network": [ + [ + "probeNetwork - default - end", + 3, + 161976 + ], + [ + "probeNetwork - default - start", + 0, + 0 + ] + ], + "gfx": [ + [ + "probeGFX - default - end", + 2, + 13, + 13, + 1, + [ + 98636, + 98636 + ], + [ + 142, + 142 + ], + [ + 1344, + 1344 + ] + ] + ] +}
\ No newline at end of file diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images/metrics.json new file mode 100644 index 0000000000..1c7d44f5fa --- /dev/null +++ b/render-test/metrics/android/render-tests/text-field/formatted-images/metrics.json @@ -0,0 +1,35 @@ +{ + "network": [ + [ + "probeNetwork - default - end", + 3, + 161976 + ], + [ + "probeNetwork - default - start", + 0, + 0 + ] + ], + "gfx": [ + [ + "probeGFX - default - end", + 3, + 4, + 17, + 1, + [ + 52510, + 52510 + ], + [ + 166, + 166 + ], + [ + 1600, + 1600 + ] + ] + ] +}
\ No newline at end of file |
