[test-runner] Make Android RenderTestRunner bot fail when test cases are not fully passed (#15994)

* [test-runner] Fix failed tests, generate android specific expectations

* [test-runner] Check render test runner testing result

* [test-runner] Add error case to android ignore list

*  [test-runner] Update metrics

* [test-runner] Move Android render tests with big diff(>0.006) to ignore list

31 files changed, 357 insertions, 31 deletions

diff --git a/circle.yml b/circle.yml
index 1290338091..2cd1aa4b37 100644
--- a/circle.yml
+++ b/circle.yml
@@ -665,6 +665,12 @@ jobs:
                 --device-ids flame --os-version-ids 29 --locales en --orientations portrait --timeout 20m \
                 --directories-to-pull /sdcard --results-dir render-test-app-${CIRCLE_BUILD_NUM} \
                 --no-record-video --no-performance-metrics
+            fi
+      - run:
+          name: Retrieve TestResult from gcloud
+          when: always
+          command: |
+            if [[ -n "${GCLOUD_SERVICE_ACCOUNT_JSON}" && -z "${SKIP_FIREBASE:-}" ]]; then
               testResult=$(gsutil ls gs://test-lab-186672a0qp5bq-ycr70axads3nc/render-test-app-${CIRCLE_BUILD_NUM}/**/*.html)
               mkdir -p /tmp/tests/render-test-app
               gsutil cp $testResult /tmp/tests/render-test-app | true
diff --git a/next/platform/android/android.cmake b/next/platform/android/android.cmake
index 0f8dd04315..29bc9c1326 100644
--- a/next/platform/android/android.cmake
+++ b/next/platform/android/android.cmake
@@ -357,29 +357,18 @@ add_custom_command(
         -E
         copy
         ${MBGL_ROOT}/platform/node/test/ignores.json
-        ${MBGL_ROOT}/ignores/ignores.json
-    COMMAND
-        ${CMAKE_COMMAND}
-        -E
-        copy
-        ${MBGL_ROOT}/render-test/linux-ignores.json
-        ${MBGL_ROOT}/ignores/linux-ignores.json
+        ${MBGL_ROOT}/render-test/ignores/ignores.json
     COMMAND
         ${CMAKE_COMMAND}
         -E
         tar
-        "chvf"
+        "chf"
         "render-test/android/app/src/main/assets/data.zip"
         --format=zip
         --files-from=render-test/android/app/src/main/assets/to_zip.txt
     COMMAND
         ${CMAKE_COMMAND}
         -E
-        remove_directory
-        ${MBGL_ROOT}/ignores
-    COMMAND
-        ${CMAKE_COMMAND}
-        -E
         remove
         ${MBGL_ROOT}/android-manifest*
     WORKING_DIRECTORY ${MBGL_ROOT}
diff --git a/platform/android/src/test/render_test_runner.cpp b/platform/android/src/test/render_test_runner.cpp
index d0af8d21a6..7e209aff3a 100644
--- a/platform/android/src/test/render_test_runner.cpp
+++ b/platform/android/src/test/render_test_runner.cpp
@@ -79,7 +79,7 @@ std::string jstringToStdString(JNIEnv* env, jstring jStr) {
     return ret;
 }
 
-void changeState(JNIEnv* env, struct android_app* app) {
+void changeState(JNIEnv* env, struct android_app* app, bool result) {
     jobject nativeActivity = app->activity->clazz;
     jclass acl = env->GetObjectClass(nativeActivity);
     jmethodID getClassLoader = env->GetMethodID(acl, "getClassLoader", "()Ljava/lang/ClassLoader;");
@@ -89,6 +89,8 @@ void changeState(JNIEnv* env, struct android_app* app) {
     JavaWrapper<jstring> strClassName(env, env->NewStringUTF("android.app.TestState"));
     jclass testStateClass = static_cast<jclass>(env->CallObjectMethod(cls, findClass, strClassName.get()));
     if (testStateClass != NULL) {
+        jfieldID id0 = env->GetStaticFieldID(testStateClass, "testResult", "Z");
+        env->SetStaticBooleanField(testStateClass, id0, result);
         jfieldID id = env->GetStaticFieldID(testStateClass, "running", "Z");
         env->SetStaticBooleanField(testStateClass, id, false);
     }
@@ -278,10 +280,11 @@ void android_main(struct android_app* app) {
             mbgl::Log::Info(mbgl::Event::General, "End running RenderTestRunner with manifest: '%s'", manifest.c_str());
             return result;
         };
-        runTestWithManifest("/android-manifest-probe-network-gfx.json");
-        runTestWithManifest("/android-manifest-probe-memory.json");
+
+        auto result = runTestWithManifest("/android-manifest-probe-network-gfx.json");
+        result = runTestWithManifest("/android-manifest-probe-memory.json") && result;
         mbgl::Log::Info(mbgl::Event::General, "All tests are finished!");
-        changeState(env, app);
+        changeState(env, app, result);
     }
     while (true) {
         ALooper_pollAll(0, &outFd, &outEvents, reinterpret_cast<void**>(&source));
diff --git a/render-test/android-manifest-probe-memory.json b/render-test/android-manifest-probe-memory.json
index a3562088f4..a90d39ee83 100644
--- a/render-test/android-manifest-probe-memory.json
+++ b/render-test/android-manifest-probe-memory.json
@@ -1,7 +1,7 @@
 {
     "base_test_path":"mapbox-gl-js/test/integration",
-    "expectation_paths":["render-test/expected"],
-    "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
+    "expectation_paths":["render-test/expected", "render-test/expected/android"],
+    "ignore_paths":["render-test/ignores/ignores.json", "render-test/ignores/linux-ignores.json", "render-test/ignores/android-temp-fail.json"],
     "vendor_path":"vendor",
     "asset_path": "mapbox-gl-js/test/integration",
     "result_path":"/sdcard/",
diff --git a/render-test/android-manifest-probe-network-gfx.json b/render-test/android-manifest-probe-network-gfx.json
index b16424d999..c7b45417d0 100644
--- a/render-test/android-manifest-probe-network-gfx.json
+++ b/render-test/android-manifest-probe-network-gfx.json
@@ -1,7 +1,7 @@
 {
     "base_test_path":"mapbox-gl-js/test/integration",
-    "expectation_paths":["render-test/expected"],
-    "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
+    "expectation_paths":["render-test/expected", "render-test/expected/android"],
+    "ignore_paths":["render-test/ignores/ignores.json", "render-test/ignores/linux-ignores.json", "render-test/ignores/android-temp-fail.json"],
     "vendor_path":"vendor",
     "asset_path": "mapbox-gl-js/test/integration",
     "result_path":"/sdcard/",
diff --git a/render-test/android-manifest.json b/render-test/android-manifest.json
index ddc91d1933..00783aba60 100644
--- a/render-test/android-manifest.json
+++ b/render-test/android-manifest.json
@@ -1,10 +1,10 @@
 {
     "base_test_path":"mapbox-gl-js/test/integration",
-    "expectation_paths":["render-test/expected"],
+    "expectation_paths":["render-test/expected", "render-test/expected/android"],
     "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
     "vendor_path":"vendor",
     "asset_path": "mapbox-gl-js/test/integration",
-    "result_path":"/sdcard/",
+    "result_path": "/sdcard/",
     "probes": ["probeGFX", "probeNetwork"],
     "metric_path":"render-test/metrics/android"
-}
\ No newline at end of file
+}  
diff --git a/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java b/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java
index 3d8c333902..d508a9d5a4 100644
--- a/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java
+++ b/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java
@@ -6,6 +6,7 @@ import androidx.test.rule.ActivityTestRule;
 
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.Assert;
 import org.junit.runner.RunWith;
 import android.util.Log;
 
@@ -24,6 +25,8 @@ public class NativeActivityTest {
             Log.v("Test", "Test is running");
             Thread.sleep(1000L);
         }
-        Log.v("Test", "End the test");
+        Log.v("Test", "All render tests are finished!");
+        Assert.assertTrue("All test cases are passed", TestState.testResult);
+
     }
 }
\ No newline at end of file
diff --git a/render-test/android/app/src/androidTest/java/android/app/TestState.java b/render-test/android/app/src/androidTest/java/android/app/TestState.java
index 44a0653fb6..b374a72afe 100644
--- a/render-test/android/app/src/androidTest/java/android/app/TestState.java
+++ b/render-test/android/app/src/androidTest/java/android/app/TestState.java
@@ -2,4 +2,5 @@ package android.app;
 
 public class TestState {
     static boolean running = true;
+    static boolean testResult = true;
 }
\ No newline at end of file
diff --git a/render-test/android/app/src/main/assets/to_zip.txt b/render-test/android/app/src/main/assets/to_zip.txt
index fbf9539583..e440a7c514 100644
--- a/render-test/android/app/src/main/assets/to_zip.txt
+++ b/render-test/android/app/src/main/assets/to_zip.txt
@@ -12,8 +12,8 @@ mapbox-gl-js/test/integration/sprites/
 vendor/mapbox-gl-styles/styles/
 vendor/mapbox-gl-styles/sprites/
 vendor/mvt-fixtures/real-world/
-ignores/
+render-test/ignores/
 render-test/metrics/android/
 render-test/expected/
 android-manifest-probe-network-gfx.json
-android-manifest-probe-memory.json
\ No newline at end of file
+android-manifest-probe-memory.json
diff --git a/render-test/expected/android/render-tests/combinations/symbol-translucent--fill-translucent/expected.png b/render-test/expected/android/render-tests/combinations/symbol-translucent--fill-translucent/expected.png
new file mode 100644
index 0000000000..6a14209d0d
--- /dev/null
+++ b/render-test/expected/android/render-tests/combinations/symbol-translucent--fill-translucent/expected.png
diff --git a/render-test/expected/android/render-tests/regressions/mapbox-gl-js#2305/expected.png b/render-test/expected/android/render-tests/regressions/mapbox-gl-js#2305/expected.png
new file mode 100644
index 0000000000..99ca724f5b
--- /dev/null
+++ b/render-test/expected/android/render-tests/regressions/mapbox-gl-js#2305/expected.png
diff --git a/render-test/expected/android/render-tests/regressions/mapbox-gl-js#5978/expected.png b/render-test/expected/android/render-tests/regressions/mapbox-gl-js#5978/expected.png
new file mode 100644
index 0000000000..37fc500689
--- /dev/null
+++ b/render-test/expected/android/render-tests/regressions/mapbox-gl-js#5978/expected.png
diff --git a/render-test/expected/android/render-tests/sprites/1x-screen-1x-pattern/expected.png b/render-test/expected/android/render-tests/sprites/1x-screen-1x-pattern/expected.png
new file mode 100644
index 0000000000..e4564aeee8
--- /dev/null
+++ b/render-test/expected/android/render-tests/sprites/1x-screen-1x-pattern/expected.png
diff --git a/render-test/expected/android/render-tests/sprites/1x-screen-2x-pattern/expected.png b/render-test/expected/android/render-tests/sprites/1x-screen-2x-pattern/expected.png
new file mode 100644
index 0000000000..8cf3bb4bda
--- /dev/null
+++ b/render-test/expected/android/render-tests/sprites/1x-screen-2x-pattern/expected.png
diff --git a/render-test/expected/android/render-tests/sprites/2x-screen-1x-pattern/expected.png b/render-test/expected/android/render-tests/sprites/2x-screen-1x-pattern/expected.png
new file mode 100644
index 0000000000..0756d83652
--- /dev/null
+++ b/render-test/expected/android/render-tests/sprites/2x-screen-1x-pattern/expected.png
diff --git a/render-test/expected/android/render-tests/sprites/2x-screen-2x-pattern/expected.png b/render-test/expected/android/render-tests/sprites/2x-screen-2x-pattern/expected.png
new file mode 100644
index 0000000000..a3b27f24a2
--- /dev/null
+++ b/render-test/expected/android/render-tests/sprites/2x-screen-2x-pattern/expected.png
diff --git a/render-test/ignores/android-temp-fail.json b/render-test/ignores/android-temp-fail.json
new file mode 100644
index 0000000000..655c697528
--- /dev/null
+++ b/render-test/ignores/android-temp-fail.json
@@ -0,0 +1,9 @@
+{
+  "render-tests/icon-text-fit/both-text-anchor-1x-image-2x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.",
+  "render-tests/icon-text-fit/both-text-anchor-2x-image-1x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.",
+  "render-tests/regressions/mapbox-gl-native#9976": "Needs to be investigated and fixed.",
+  "render-tests/regressions/mapbox-gl-js#5911a": "Needs to be investigated and fixed.",
+  "render-tests/line-pattern/overscaled": "Needs to be investigated and fixed.",
+  "render-tests/runtime-styling/image-add-pattern": "Needs to be investigated and fixed.",
+  "render-tests/runtime-styling/image-update-pattern": "Needs to be investigated and fixed."
+}
diff --git a/render-test/linux-ignores.json b/render-test/ignores/linux-ignores.json
index d5a1a85c6c..d5a1a85c6c 100644
--- a/render-test/linux-ignores.json
+++ b/render-test/ignores/linux-ignores.json
diff --git a/render-test/linux-manifest-probe-gfx-network.json b/render-test/linux-manifest-probe-gfx-network.json
index b63013ca18..fedf9fa6bd 100644
--- a/render-test/linux-manifest-probe-gfx-network.json
+++ b/render-test/linux-manifest-probe-gfx-network.json
@@ -1,7 +1,7 @@
 {
     "base_test_path":"../mapbox-gl-js/test/integration",
     "expectation_paths":["expected/"],
-    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
+    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
     "vendor_path":"../vendor",
     "asset_path": "../mapbox-gl-js/test/integration",
     "probes": ["probeGFX", "probeNetwork"],
diff --git a/render-test/linux-manifest-probe-memory.json b/render-test/linux-manifest-probe-memory.json
index 25b59e4a44..e21326875d 100644
--- a/render-test/linux-manifest-probe-memory.json
+++ b/render-test/linux-manifest-probe-memory.json
@@ -1,7 +1,7 @@
 {
     "base_test_path":"../mapbox-gl-js/test/integration",
     "expectation_paths":["expected/"],
-    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
+    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
     "vendor_path":"../vendor",
     "asset_path": "../mapbox-gl-js/test/integration",
     "probes": ["probeMemory"],
diff --git a/render-test/linux-manifest.json b/render-test/linux-manifest.json
index cc72a7446c..91b062c2cc 100644
--- a/render-test/linux-manifest.json
+++ b/render-test/linux-manifest.json
@@ -1,7 +1,7 @@
 {
     "base_test_path":"../mapbox-gl-js/test/integration",
     "expectation_paths":["expected/"],
-    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
+    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
     "vendor_path":"../vendor",
     "asset_path": "../mapbox-gl-js/test/integration"
 }
\ No newline at end of file
diff --git a/render-test/linux-query-manifest.json b/render-test/linux-query-manifest.json
index a6723f5519..6d2071b19f 100644
--- a/render-test/linux-query-manifest.json
+++ b/render-test/linux-query-manifest.json
@@ -1,6 +1,6 @@
 {
     "base_test_path":"../mapbox-gl-js/test/integration",
-    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
+    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
     "vendor_path":"../vendor",
     "asset_path": "../mapbox-gl-js/test/integration"
 }
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/line-dasharray/round/segments/metrics.json b/render-test/metrics/android/render-tests/line-dasharray/round/segments/metrics.json
new file mode 100644
index 0000000000..d3af311705
--- /dev/null
+++ b/render-test/metrics/android/render-tests/line-dasharray/round/segments/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            0,
+            0
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            2,
+            4,
+            9,
+            1,
+            [
+                36864,
+                36864
+            ],
+            [
+                202,
+                202
+            ],
+            [
+                704,
+                704
+            ]
+        ]
+    ]
+}
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/line-dasharray/round/zero-gap-width/metrics.json b/render-test/metrics/android/render-tests/line-dasharray/round/zero-gap-width/metrics.json
new file mode 100644
index 0000000000..d3af311705
--- /dev/null
+++ b/render-test/metrics/android/render-tests/line-dasharray/round/zero-gap-width/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            0,
+            0
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            2,
+            4,
+            9,
+            1,
+            [
+                36864,
+                36864
+            ],
+            [
+                202,
+                202
+            ],
+            [
+                704,
+                704
+            ]
+        ]
+    ]
+}
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-constant-size/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-constant-size/metrics.json
new file mode 100644
index 0000000000..1ebdc9b090
--- /dev/null
+++ b/render-test/metrics/android/render-tests/text-field/formatted-images-constant-size/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            3,
+            161976
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            2,
+            4,
+            13,
+            1,
+            [
+                24580,
+                24580
+            ],
+            [
+                190,
+                190
+            ],
+            [
+                1856,
+                1856
+            ]
+        ]
+    ]
+}
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-line/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-line/metrics.json
new file mode 100644
index 0000000000..f7f0edc2f0
--- /dev/null
+++ b/render-test/metrics/android/render-tests/text-field/formatted-images-line/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            3,
+            120865
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            3,
+            4,
+            11,
+            1,
+            [
+                28318,
+                28318
+            ],
+            [
+                190,
+                190
+            ],
+            [
+                1424,
+                1424
+            ]
+        ]
+    ]
+}
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-multiline/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-multiline/metrics.json
new file mode 100644
index 0000000000..5dfadecdc9
--- /dev/null
+++ b/render-test/metrics/android/render-tests/text-field/formatted-images-multiline/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            3,
+            120865
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            2,
+            6,
+            13,
+            1,
+            [
+                44921,
+                44921
+            ],
+            [
+                310,
+                310
+            ],
+            [
+                3136,
+                3136
+            ]
+        ]
+    ]
+}
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-variable-anchors-justification/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-variable-anchors-justification/metrics.json
new file mode 100644
index 0000000000..312dedfbf0
--- /dev/null
+++ b/render-test/metrics/android/render-tests/text-field/formatted-images-variable-anchors-justification/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            3,
+            161976
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            1,
+            4,
+            9,
+            1,
+            [
+                24580,
+                24580
+            ],
+            [
+                2290,
+                2290
+            ],
+            [
+                24256,
+                24256
+            ]
+        ]
+    ]
+}
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-vertical/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-vertical/metrics.json
new file mode 100644
index 0000000000..e4a8816967
--- /dev/null
+++ b/render-test/metrics/android/render-tests/text-field/formatted-images-vertical/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            4,
+            255564
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            1,
+            4,
+            9,
+            1,
+            [
+                35756,
+                35756
+            ],
+            [
+                310,
+                310
+            ],
+            [
+                3136,
+                3136
+            ]
+        ]
+    ]
+}
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-zoom-dependent-size/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-zoom-dependent-size/metrics.json
new file mode 100644
index 0000000000..7cf8a26c5d
--- /dev/null
+++ b/render-test/metrics/android/render-tests/text-field/formatted-images-zoom-dependent-size/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            3,
+            161976
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            2,
+            13,
+            13,
+            1,
+            [
+                98636,
+                98636
+            ],
+            [
+                142,
+                142
+            ],
+            [
+                1344,
+                1344
+            ]
+        ]
+    ]
+}
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images/metrics.json
new file mode 100644
index 0000000000..1c7d44f5fa
--- /dev/null
+++ b/render-test/metrics/android/render-tests/text-field/formatted-images/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            3,
+            161976
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            3,
+            4,
+            17,
+            1,
+            [
+                52510,
+                52510
+            ],
+            [
+                166,
+                166
+            ],
+            [
+                1600,
+                1600
+            ]
+        ]
+    ]
+}
\ No newline at end of file