[test-runner] Make Android RenderTestRunner bot fail when test cases are not fully passed (#15994)

* [test-runner] Fix failed tests, generate android specific expectations

* [test-runner] Check render test runner testing result

* [test-runner] Add error case to android ignore list

*  [test-runner] Update metrics

* [test-runner] Move Android render tests with big diff(>0.006) to ignore list

28 files changed, 342 insertions, 14 deletions

diff --git a/render-test/android-manifest-probe-memory.json b/render-test/android-manifest-probe-memory.json
index a3562088f4..a90d39ee83 100644
--- a/render-test/android-manifest-probe-memory.json
+++ b/render-test/android-manifest-probe-memory.json
@@ -1,7 +1,7 @@
 {
     "base_test_path":"mapbox-gl-js/test/integration",
-    "expectation_paths":["render-test/expected"],
-    "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
+    "expectation_paths":["render-test/expected", "render-test/expected/android"],
+    "ignore_paths":["render-test/ignores/ignores.json", "render-test/ignores/linux-ignores.json", "render-test/ignores/android-temp-fail.json"],
     "vendor_path":"vendor",
     "asset_path": "mapbox-gl-js/test/integration",
     "result_path":"/sdcard/",
diff --git a/render-test/android-manifest-probe-network-gfx.json b/render-test/android-manifest-probe-network-gfx.json
index b16424d999..c7b45417d0 100644
--- a/render-test/android-manifest-probe-network-gfx.json
+++ b/render-test/android-manifest-probe-network-gfx.json
@@ -1,7 +1,7 @@
 {
     "base_test_path":"mapbox-gl-js/test/integration",
-    "expectation_paths":["render-test/expected"],
-    "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
+    "expectation_paths":["render-test/expected", "render-test/expected/android"],
+    "ignore_paths":["render-test/ignores/ignores.json", "render-test/ignores/linux-ignores.json", "render-test/ignores/android-temp-fail.json"],
     "vendor_path":"vendor",
     "asset_path": "mapbox-gl-js/test/integration",
     "result_path":"/sdcard/",
diff --git a/render-test/android-manifest.json b/render-test/android-manifest.json
index ddc91d1933..00783aba60 100644
--- a/render-test/android-manifest.json
+++ b/render-test/android-manifest.json
@@ -1,10 +1,10 @@
 {
     "base_test_path":"mapbox-gl-js/test/integration",
-    "expectation_paths":["render-test/expected"],
+    "expectation_paths":["render-test/expected", "render-test/expected/android"],
     "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
     "vendor_path":"vendor",
     "asset_path": "mapbox-gl-js/test/integration",
-    "result_path":"/sdcard/",
+    "result_path": "/sdcard/",
     "probes": ["probeGFX", "probeNetwork"],
     "metric_path":"render-test/metrics/android"
-}
\ No newline at end of file
+}  
diff --git a/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java b/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java
index 3d8c333902..d508a9d5a4 100644
--- a/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java
+++ b/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java
@@ -6,6 +6,7 @@ import androidx.test.rule.ActivityTestRule;
 
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.Assert;
 import org.junit.runner.RunWith;
 import android.util.Log;
 
@@ -24,6 +25,8 @@ public class NativeActivityTest {
             Log.v("Test", "Test is running");
             Thread.sleep(1000L);
         }
-        Log.v("Test", "End the test");
+        Log.v("Test", "All render tests are finished!");
+        Assert.assertTrue("All test cases are passed", TestState.testResult);
+
     }
 }
\ No newline at end of file
diff --git a/render-test/android/app/src/androidTest/java/android/app/TestState.java b/render-test/android/app/src/androidTest/java/android/app/TestState.java
index 44a0653fb6..b374a72afe 100644
--- a/render-test/android/app/src/androidTest/java/android/app/TestState.java
+++ b/render-test/android/app/src/androidTest/java/android/app/TestState.java
@@ -2,4 +2,5 @@ package android.app;
 
 public class TestState {
     static boolean running = true;
+    static boolean testResult = true;
 }
\ No newline at end of file
diff --git a/render-test/android/app/src/main/assets/to_zip.txt b/render-test/android/app/src/main/assets/to_zip.txt
index fbf9539583..e440a7c514 100644
--- a/render-test/android/app/src/main/assets/to_zip.txt
+++ b/render-test/android/app/src/main/assets/to_zip.txt
@@ -12,8 +12,8 @@ mapbox-gl-js/test/integration/sprites/
 vendor/mapbox-gl-styles/styles/
 vendor/mapbox-gl-styles/sprites/
 vendor/mvt-fixtures/real-world/
-ignores/
+render-test/ignores/
 render-test/metrics/android/
 render-test/expected/
 android-manifest-probe-network-gfx.json
-android-manifest-probe-memory.json
\ No newline at end of file
+android-manifest-probe-memory.json
diff --git a/render-test/expected/android/render-tests/combinations/symbol-translucent--fill-translucent/expected.png b/render-test/expected/android/render-tests/combinations/symbol-translucent--fill-translucent/expected.png
new file mode 100644
index 0000000000..6a14209d0d
--- /dev/null
+++ b/render-test/expected/android/render-tests/combinations/symbol-translucent--fill-translucent/expected.png
diff --git a/render-test/expected/android/render-tests/regressions/mapbox-gl-js#2305/expected.png b/render-test/expected/android/render-tests/regressions/mapbox-gl-js#2305/expected.png
new file mode 100644
index 0000000000..99ca724f5b
--- /dev/null
+++ b/render-test/expected/android/render-tests/regressions/mapbox-gl-js#2305/expected.png
diff --git a/render-test/expected/android/render-tests/regressions/mapbox-gl-js#5978/expected.png b/render-test/expected/android/render-tests/regressions/mapbox-gl-js#5978/expected.png
new file mode 100644
index 0000000000..37fc500689
--- /dev/null
+++ b/render-test/expected/android/render-tests/regressions/mapbox-gl-js#5978/expected.png
diff --git a/render-test/expected/android/render-tests/sprites/1x-screen-1x-pattern/expected.png b/render-test/expected/android/render-tests/sprites/1x-screen-1x-pattern/expected.png
new file mode 100644
index 0000000000..e4564aeee8
--- /dev/null
+++ b/render-test/expected/android/render-tests/sprites/1x-screen-1x-pattern/expected.png
diff --git a/render-test/expected/android/render-tests/sprites/1x-screen-2x-pattern/expected.png b/render-test/expected/android/render-tests/sprites/1x-screen-2x-pattern/expected.png
new file mode 100644
index 0000000000..8cf3bb4bda
--- /dev/null
+++ b/render-test/expected/android/render-tests/sprites/1x-screen-2x-pattern/expected.png
diff --git a/render-test/expected/android/render-tests/sprites/2x-screen-1x-pattern/expected.png b/render-test/expected/android/render-tests/sprites/2x-screen-1x-pattern/expected.png
new file mode 100644
index 0000000000..0756d83652
--- /dev/null
+++ b/render-test/expected/android/render-tests/sprites/2x-screen-1x-pattern/expected.png
diff --git a/render-test/expected/android/render-tests/sprites/2x-screen-2x-pattern/expected.png b/render-test/expected/android/render-tests/sprites/2x-screen-2x-pattern/expected.png
new file mode 100644
index 0000000000..a3b27f24a2
--- /dev/null
+++ b/render-test/expected/android/render-tests/sprites/2x-screen-2x-pattern/expected.png
diff --git a/render-test/ignores/android-temp-fail.json b/render-test/ignores/android-temp-fail.json
new file mode 100644
index 0000000000..655c697528
--- /dev/null
+++ b/render-test/ignores/android-temp-fail.json
@@ -0,0 +1,9 @@
+{
+  "render-tests/icon-text-fit/both-text-anchor-1x-image-2x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.",
+  "render-tests/icon-text-fit/both-text-anchor-2x-image-1x-screen": "Temporarily fails as android test app does not derefence symbolinks in sprites folder.",
+  "render-tests/regressions/mapbox-gl-native#9976": "Needs to be investigated and fixed.",
+  "render-tests/regressions/mapbox-gl-js#5911a": "Needs to be investigated and fixed.",
+  "render-tests/line-pattern/overscaled": "Needs to be investigated and fixed.",
+  "render-tests/runtime-styling/image-add-pattern": "Needs to be investigated and fixed.",
+  "render-tests/runtime-styling/image-update-pattern": "Needs to be investigated and fixed."
+}
diff --git a/render-test/linux-ignores.json b/render-test/ignores/linux-ignores.json
index d5a1a85c6c..d5a1a85c6c 100644
--- a/render-test/linux-ignores.json
+++ b/render-test/ignores/linux-ignores.json
diff --git a/render-test/linux-manifest-probe-gfx-network.json b/render-test/linux-manifest-probe-gfx-network.json
index b63013ca18..fedf9fa6bd 100644
--- a/render-test/linux-manifest-probe-gfx-network.json
+++ b/render-test/linux-manifest-probe-gfx-network.json
@@ -1,7 +1,7 @@
 {
     "base_test_path":"../mapbox-gl-js/test/integration",
     "expectation_paths":["expected/"],
-    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
+    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
     "vendor_path":"../vendor",
     "asset_path": "../mapbox-gl-js/test/integration",
     "probes": ["probeGFX", "probeNetwork"],
diff --git a/render-test/linux-manifest-probe-memory.json b/render-test/linux-manifest-probe-memory.json
index 25b59e4a44..e21326875d 100644
--- a/render-test/linux-manifest-probe-memory.json
+++ b/render-test/linux-manifest-probe-memory.json
@@ -1,7 +1,7 @@
 {
     "base_test_path":"../mapbox-gl-js/test/integration",
     "expectation_paths":["expected/"],
-    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
+    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
     "vendor_path":"../vendor",
     "asset_path": "../mapbox-gl-js/test/integration",
     "probes": ["probeMemory"],
diff --git a/render-test/linux-manifest.json b/render-test/linux-manifest.json
index cc72a7446c..91b062c2cc 100644
--- a/render-test/linux-manifest.json
+++ b/render-test/linux-manifest.json
@@ -1,7 +1,7 @@
 {
     "base_test_path":"../mapbox-gl-js/test/integration",
     "expectation_paths":["expected/"],
-    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
+    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
     "vendor_path":"../vendor",
     "asset_path": "../mapbox-gl-js/test/integration"
 }
\ No newline at end of file
diff --git a/render-test/linux-query-manifest.json b/render-test/linux-query-manifest.json
index a6723f5519..6d2071b19f 100644
--- a/render-test/linux-query-manifest.json
+++ b/render-test/linux-query-manifest.json
@@ -1,6 +1,6 @@
 {
     "base_test_path":"../mapbox-gl-js/test/integration",
-    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
+    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/ignores/linux-ignores.json"],
     "vendor_path":"../vendor",
     "asset_path": "../mapbox-gl-js/test/integration"
 }
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/line-dasharray/round/segments/metrics.json b/render-test/metrics/android/render-tests/line-dasharray/round/segments/metrics.json
new file mode 100644
index 0000000000..d3af311705
--- /dev/null
+++ b/render-test/metrics/android/render-tests/line-dasharray/round/segments/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            0,
+            0
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            2,
+            4,
+            9,
+            1,
+            [
+                36864,
+                36864
+            ],
+            [
+                202,
+                202
+            ],
+            [
+                704,
+                704
+            ]
+        ]
+    ]
+}
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/line-dasharray/round/zero-gap-width/metrics.json b/render-test/metrics/android/render-tests/line-dasharray/round/zero-gap-width/metrics.json
new file mode 100644
index 0000000000..d3af311705
--- /dev/null
+++ b/render-test/metrics/android/render-tests/line-dasharray/round/zero-gap-width/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            0,
+            0
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            2,
+            4,
+            9,
+            1,
+            [
+                36864,
+                36864
+            ],
+            [
+                202,
+                202
+            ],
+            [
+                704,
+                704
+            ]
+        ]
+    ]
+}
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-constant-size/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-constant-size/metrics.json
new file mode 100644
index 0000000000..1ebdc9b090
--- /dev/null
+++ b/render-test/metrics/android/render-tests/text-field/formatted-images-constant-size/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            3,
+            161976
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            2,
+            4,
+            13,
+            1,
+            [
+                24580,
+                24580
+            ],
+            [
+                190,
+                190
+            ],
+            [
+                1856,
+                1856
+            ]
+        ]
+    ]
+}
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-line/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-line/metrics.json
new file mode 100644
index 0000000000..f7f0edc2f0
--- /dev/null
+++ b/render-test/metrics/android/render-tests/text-field/formatted-images-line/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            3,
+            120865
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            3,
+            4,
+            11,
+            1,
+            [
+                28318,
+                28318
+            ],
+            [
+                190,
+                190
+            ],
+            [
+                1424,
+                1424
+            ]
+        ]
+    ]
+}
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-multiline/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-multiline/metrics.json
new file mode 100644
index 0000000000..5dfadecdc9
--- /dev/null
+++ b/render-test/metrics/android/render-tests/text-field/formatted-images-multiline/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            3,
+            120865
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            2,
+            6,
+            13,
+            1,
+            [
+                44921,
+                44921
+            ],
+            [
+                310,
+                310
+            ],
+            [
+                3136,
+                3136
+            ]
+        ]
+    ]
+}
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-variable-anchors-justification/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-variable-anchors-justification/metrics.json
new file mode 100644
index 0000000000..312dedfbf0
--- /dev/null
+++ b/render-test/metrics/android/render-tests/text-field/formatted-images-variable-anchors-justification/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            3,
+            161976
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            1,
+            4,
+            9,
+            1,
+            [
+                24580,
+                24580
+            ],
+            [
+                2290,
+                2290
+            ],
+            [
+                24256,
+                24256
+            ]
+        ]
+    ]
+}
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-vertical/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-vertical/metrics.json
new file mode 100644
index 0000000000..e4a8816967
--- /dev/null
+++ b/render-test/metrics/android/render-tests/text-field/formatted-images-vertical/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            4,
+            255564
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            1,
+            4,
+            9,
+            1,
+            [
+                35756,
+                35756
+            ],
+            [
+                310,
+                310
+            ],
+            [
+                3136,
+                3136
+            ]
+        ]
+    ]
+}
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images-zoom-dependent-size/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images-zoom-dependent-size/metrics.json
new file mode 100644
index 0000000000..7cf8a26c5d
--- /dev/null
+++ b/render-test/metrics/android/render-tests/text-field/formatted-images-zoom-dependent-size/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            3,
+            161976
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            2,
+            13,
+            13,
+            1,
+            [
+                98636,
+                98636
+            ],
+            [
+                142,
+                142
+            ],
+            [
+                1344,
+                1344
+            ]
+        ]
+    ]
+}
\ No newline at end of file
diff --git a/render-test/metrics/android/render-tests/text-field/formatted-images/metrics.json b/render-test/metrics/android/render-tests/text-field/formatted-images/metrics.json
new file mode 100644
index 0000000000..1c7d44f5fa
--- /dev/null
+++ b/render-test/metrics/android/render-tests/text-field/formatted-images/metrics.json
@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            3,
+            161976
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            3,
+            4,
+            17,
+            1,
+            [
+                52510,
+                52510
+            ],
+            [
+                166,
+                166
+            ],
+            [
+                1600,
+                1600
+            ]
+        ]
+    ]
+}
\ No newline at end of file