[test-runner] Make Android RenderTestRunner bot fail when test cases are not fully passed (#15994)

* [test-runner] Fix failed tests, generate android specific expectations

* [test-runner] Check render test runner testing result

* [test-runner] Add error case to android ignore list

*  [test-runner] Update metrics

* [test-runner] Move Android render tests with big diff(>0.006) to ignore list

1 files changed, 7 insertions, 4 deletions

diff --git a/platform/android/src/test/render_test_runner.cpp b/platform/android/src/test/render_test_runner.cpp
index d0af8d21a6..7e209aff3a 100644
--- a/platform/android/src/test/render_test_runner.cpp
+++ b/platform/android/src/test/render_test_runner.cpp
@@ -79,7 +79,7 @@ std::string jstringToStdString(JNIEnv* env, jstring jStr) {
     return ret;
 }
 
-void changeState(JNIEnv* env, struct android_app* app) {
+void changeState(JNIEnv* env, struct android_app* app, bool result) {
     jobject nativeActivity = app->activity->clazz;
     jclass acl = env->GetObjectClass(nativeActivity);
     jmethodID getClassLoader = env->GetMethodID(acl, "getClassLoader", "()Ljava/lang/ClassLoader;");
@@ -89,6 +89,8 @@ void changeState(JNIEnv* env, struct android_app* app) {
     JavaWrapper<jstring> strClassName(env, env->NewStringUTF("android.app.TestState"));
     jclass testStateClass = static_cast<jclass>(env->CallObjectMethod(cls, findClass, strClassName.get()));
     if (testStateClass != NULL) {
+        jfieldID id0 = env->GetStaticFieldID(testStateClass, "testResult", "Z");
+        env->SetStaticBooleanField(testStateClass, id0, result);
         jfieldID id = env->GetStaticFieldID(testStateClass, "running", "Z");
         env->SetStaticBooleanField(testStateClass, id, false);
     }
@@ -278,10 +280,11 @@ void android_main(struct android_app* app) {
             mbgl::Log::Info(mbgl::Event::General, "End running RenderTestRunner with manifest: '%s'", manifest.c_str());
             return result;
         };
-        runTestWithManifest("/android-manifest-probe-network-gfx.json");
-        runTestWithManifest("/android-manifest-probe-memory.json");
+
+        auto result = runTestWithManifest("/android-manifest-probe-network-gfx.json");
+        result = runTestWithManifest("/android-manifest-probe-memory.json") && result;
         mbgl::Log::Info(mbgl::Event::General, "All tests are finished!");
-        changeState(env, app);
+        changeState(env, app, result);
     }
     while (true) {
         ALooper_pollAll(0, &outFd, &outEvents, reinterpret_cast<void**>(&source));