[test-runner]check render test runner testing result

11 files changed, 207 insertions, 194 deletions

diff --git a/circle.yml b/circle.yml
index 1169cacfb9..caf80420bd 100644
--- a/circle.yml
+++ b/circle.yml
@@ -4,192 +4,192 @@ workflows:
   version: 2
   mbgl-next:
     jobs:
-      # - next-sanity-checks
+      - next-sanity-checks
       - next-android-render-test-runner
-  #     - next-build-template:
-  #         name: next-android-armeabi-v7a-release
-  #         executor_name: ubuntu-disco
-  #         target_is_android: true
-  #         config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_TOOLCHAIN_FILE=/opt/android/ndk-bundle/build/cmake/android.toolchain.cmake -DANDROID_CCACHE=/usr/bin/ccache -DANDROID_ABI=armeabi-v7a'
-  #         install: true
-  #     - next-build-template:
-  #         name: next-android-arm64-v8a-release
-  #         executor_name: ubuntu-disco
-  #         target_is_android: true
-  #         config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_TOOLCHAIN_FILE=/opt/android/ndk-bundle/build/cmake/android.toolchain.cmake -DANDROID_CCACHE=/usr/bin/ccache -DANDROID_ABI=arm64-v8a'
-  #         install: true
-  #     - next-build-template:
-  #         name: next-android-x86-release
-  #         executor_name: ubuntu-disco
-  #         target_is_android: true
-  #         config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_TOOLCHAIN_FILE=/opt/android/ndk-bundle/build/cmake/android.toolchain.cmake -DANDROID_CCACHE=/usr/bin/ccache -DANDROID_ABI=x86'
-  #         install: true
-  #     - next-build-template:
-  #         name: next-android-x86_64-release
-  #         executor_name: ubuntu-disco
-  #         target_is_android: true
-  #         config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_TOOLCHAIN_FILE=/opt/android/ndk-bundle/build/cmake/android.toolchain.cmake -DANDROID_CCACHE=/usr/bin/ccache -DANDROID_ABI=x86_64'
-  #         install: true
-  #     - next-build-template:
-  #         name: next-linux-gcc8-release
-  #         executor_name: ubuntu-disco
-  #         target_is_linux: true
-  #         config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8'
-  #         install: true
-  #     - next-build-template:
-  #         name: next-linux-gcc4.9-release
-  #         executor_name: ubuntu-disco
-  #         target_is_linux: true
-  #         requires:
-  #           - next-linux-gcc8-release
-  #         config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=gcc-4.9 -DCMAKE_CXX_COMPILER=g++-4.9'
-  #         build_params: '--target mbgl-glfw'
-  #         test_params: '-N -Q'
-  #     - next-build-template:
-  #         name: next-linux-gcc8-debug-coverage
-  #         executor_name: ubuntu-disco
-  #         target_is_linux: true
-  #         requires:
-  #           - next-linux-gcc8-release
-  #         config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 -DCMAKE_BUILD_TYPE=DebugCoverage'
-  #     - next-build-template:
-  #         name: next-linux-clang8-release
-  #         executor_name: ubuntu-disco
-  #         target_is_linux: true
-  #         requires:
-  #           - next-android-armeabi-v7a-release
-  #           - next-android-arm64-v8a-release
-  #           - next-android-x86-release
-  #           - next-android-x86_64-release
-  #           - next-linux-gcc8-release
-  #           - next-macos-xcode11-release
-  #         config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8'
-  #         install: true
-  #         binary_size_metrics: true
-  #     - next-build-template:
-  #         name: next-FIXME-linux-asan
-  #         executor_name: ubuntu-disco
-  #         target_is_linux: true
-  #         requires:
-  #           - next-sanity-checks
-  #         config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8 -DCMAKE_BUILD_TYPE=Sanitize -DMBGL_WITH_SANITIZER=address'
-  #         test_params: '|| true'
-  #     - next-build-template:
-  #         name: next-linux-tsan
-  #         executor_name: ubuntu-disco
-  #         target_is_linux: true
-  #         requires:
-  #           - next-sanity-checks
-  #         config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8 -DCMAKE_BUILD_TYPE=Sanitize -DMBGL_WITH_SANITIZER=thread'
-  #     - next-build-template:
-  #         name: next-FIXME-linux-valgrind
-  #         executor_name: ubuntu-disco
-  #         target_is_linux: true
-  #         config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8 -DCMAKE_CXX_FLAGS=-DSANITIZE'
-  #         test_params: '-j 4 -E "mbgl-node|gl-benchmark" -D ExperimentalMemCheck || true'
-  #     - next-build-template:
-  #         name: next-FIXME-linux-ubsan
-  #         executor_name: ubuntu-disco
-  #         target_is_linux: true
-  #         requires:
-  #           - next-sanity-checks
-  #         config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8 -DCMAKE_BUILD_TYPE=Sanitize -DMBGL_WITH_SANITIZER=undefined'
-  #         test_params: '|| true'
-  #     - next-build-template:
-  #         name: next-qt5-linux-gcc5-release
-  #         executor_name: ubuntu-disco
-  #         target_is_linux: true
-  #         config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=gcc-5 -DCMAKE_CXX_COMPILER=g++-5 -DMBGL_WITH_QT=ON'
-  #     - next-build-template:
-  #         name: next-qt5-macos-gcc5-release
-  #         executor_name: macos-11_0_0
-  #         target_is_macos: true
-  #         requires:
-  #           - next-qt5-linux-gcc5-release
-  #         config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DMBGL_WITH_QT=ON -DCMAKE_PREFIX_PATH=$(echo /usr/local/Cellar/qt/5.*/lib/cmake)'
-  #         test_params: '-N -Q'
-  #     - next-build-template:
-  #         name: next-macos-xcode11-release
-  #         executor_name: macos-11_0_0
-  #         target_is_macos: true
-  #         config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache'
-  #         install: true
-  #     - next-build-template:
-  #         name: next-macos-xcode11-debug
-  #         executor_name: macos-11_0_0
-  #         target_is_macos: true
-  #         requires:
-  #           - next-macos-xcode11-release
-  #         config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=Debug'
-  #     - next-build-template:
-  #         name: next-ios-xcode11-release
-  #         executor_name: macos-11_0_0
-  #         target_is_macos: true
-  #         config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_SYSTEM_NAME=iOS -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_SYSROOT=iphonesimulator'
-  #         test_params: '-Q -N'
-  # mbgl-legacy:
-  #   jobs:
-  #     #
-  #     # Naming convention: {platform}-{additional description}-{build type}
-  #     #   - {platform} is the client platform/framework, which may differ from
-  #     #     the build platform. Specify both if applicable, e.g., "qt5-macos".
-  #     #   - {additional description} optionally describes the compiler or other
-  #     #     unique aspect of the build environment.
-  #     #   - {build type} is typically "debug" or "release".
-  #     #
-  #     - android-debug-arm-v7-buck
-  #     - android-arm-template:
-  #         name: android-debug-arm-v8
-  #     - android-arm-template:
-  #         name: android-gnustl-arm-v7
-  #         stl: gnustl_shared
-  #         firebase_device_id: "flo"
-  #         firebase_device_os: "21"
-  #         image: android-ndk-r17c:1d5db0eb34
-  #         abi: arm-v7
-  #     - android-release:
-  #         filters:
-  #           tags:
-  #             only: /android-v.*/
-  #     - node-linux-release:
-  #         filters:
-  #           tags:
-  #             only: /node-.*/
-  #     - node-macos-release:
-  #         filters:
-  #           tags:
-  #             only: /node-.*/
-  #     - linux-clang-7-sanitize-address-undefined
-  #     - linux-clang-7-sanitize-thread
-  #     - linux-gcc5-debug-coverage
-  #     - linux-doxygen
-  #     - ios-debug
-  #     - ios-debug-xcode10
-  #     - ios-release-template:
-  #         name: ios-release
-  #     - ios-release-tag:
-  #         filters:
-  #           tags:
-  #             only: /ios-.*/
-  #           branches:
-  #             ignore: /.*/
-  #     - macos-debug
-  # nightly:
-  #   triggers:
-  #     - schedule:
-  #         cron: "0 5 * * *"
-  #         filters:
-  #           branches:
-  #             only:
-  #               - master
-  #   jobs:
-  #     - metrics-nightly
-  #     - ios-release-template:
-  #         name: ios-release-nightly
-  #     - ios-sanitize-nightly
-  #     - ios-sanitize-address-nightly
-  #     - ios-static-analyzer-nightly
-  #     - ios-static-analyzer-nightly-xcode10
+      - next-build-template:
+          name: next-android-armeabi-v7a-release
+          executor_name: ubuntu-disco
+          target_is_android: true
+          config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_TOOLCHAIN_FILE=/opt/android/ndk-bundle/build/cmake/android.toolchain.cmake -DANDROID_CCACHE=/usr/bin/ccache -DANDROID_ABI=armeabi-v7a'
+          install: true
+      - next-build-template:
+          name: next-android-arm64-v8a-release
+          executor_name: ubuntu-disco
+          target_is_android: true
+          config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_TOOLCHAIN_FILE=/opt/android/ndk-bundle/build/cmake/android.toolchain.cmake -DANDROID_CCACHE=/usr/bin/ccache -DANDROID_ABI=arm64-v8a'
+          install: true
+      - next-build-template:
+          name: next-android-x86-release
+          executor_name: ubuntu-disco
+          target_is_android: true
+          config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_TOOLCHAIN_FILE=/opt/android/ndk-bundle/build/cmake/android.toolchain.cmake -DANDROID_CCACHE=/usr/bin/ccache -DANDROID_ABI=x86'
+          install: true
+      - next-build-template:
+          name: next-android-x86_64-release
+          executor_name: ubuntu-disco
+          target_is_android: true
+          config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_TOOLCHAIN_FILE=/opt/android/ndk-bundle/build/cmake/android.toolchain.cmake -DANDROID_CCACHE=/usr/bin/ccache -DANDROID_ABI=x86_64'
+          install: true
+      - next-build-template:
+          name: next-linux-gcc8-release
+          executor_name: ubuntu-disco
+          target_is_linux: true
+          config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8'
+          install: true
+      - next-build-template:
+          name: next-linux-gcc4.9-release
+          executor_name: ubuntu-disco
+          target_is_linux: true
+          requires:
+            - next-linux-gcc8-release
+          config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=gcc-4.9 -DCMAKE_CXX_COMPILER=g++-4.9'
+          build_params: '--target mbgl-glfw'
+          test_params: '-N -Q'
+      - next-build-template:
+          name: next-linux-gcc8-debug-coverage
+          executor_name: ubuntu-disco
+          target_is_linux: true
+          requires:
+            - next-linux-gcc8-release
+          config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8 -DCMAKE_BUILD_TYPE=DebugCoverage'
+      - next-build-template:
+          name: next-linux-clang8-release
+          executor_name: ubuntu-disco
+          target_is_linux: true
+          requires:
+            - next-android-armeabi-v7a-release
+            - next-android-arm64-v8a-release
+            - next-android-x86-release
+            - next-android-x86_64-release
+            - next-linux-gcc8-release
+            - next-macos-xcode11-release
+          config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8'
+          install: true
+          binary_size_metrics: true
+      - next-build-template:
+          name: next-FIXME-linux-asan
+          executor_name: ubuntu-disco
+          target_is_linux: true
+          requires:
+            - next-sanity-checks
+          config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8 -DCMAKE_BUILD_TYPE=Sanitize -DMBGL_WITH_SANITIZER=address'
+          test_params: '|| true'
+      - next-build-template:
+          name: next-linux-tsan
+          executor_name: ubuntu-disco
+          target_is_linux: true
+          requires:
+            - next-sanity-checks
+          config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8 -DCMAKE_BUILD_TYPE=Sanitize -DMBGL_WITH_SANITIZER=thread'
+      - next-build-template:
+          name: next-FIXME-linux-valgrind
+          executor_name: ubuntu-disco
+          target_is_linux: true
+          config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8 -DCMAKE_CXX_FLAGS=-DSANITIZE'
+          test_params: '-j 4 -E "mbgl-node|gl-benchmark" -D ExperimentalMemCheck || true'
+      - next-build-template:
+          name: next-FIXME-linux-ubsan
+          executor_name: ubuntu-disco
+          target_is_linux: true
+          requires:
+            - next-sanity-checks
+          config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-8 -DCMAKE_CXX_COMPILER=clang++-8 -DCMAKE_BUILD_TYPE=Sanitize -DMBGL_WITH_SANITIZER=undefined'
+          test_params: '|| true'
+      - next-build-template:
+          name: next-qt5-linux-gcc5-release
+          executor_name: ubuntu-disco
+          target_is_linux: true
+          config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=gcc-5 -DCMAKE_CXX_COMPILER=g++-5 -DMBGL_WITH_QT=ON'
+      - next-build-template:
+          name: next-qt5-macos-gcc5-release
+          executor_name: macos-11_0_0
+          target_is_macos: true
+          requires:
+            - next-qt5-linux-gcc5-release
+          config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DMBGL_WITH_QT=ON -DCMAKE_PREFIX_PATH=$(echo /usr/local/Cellar/qt/5.*/lib/cmake)'
+          test_params: '-N -Q'
+      - next-build-template:
+          name: next-macos-xcode11-release
+          executor_name: macos-11_0_0
+          target_is_macos: true
+          config_params: '-G Ninja -DCMAKE_INSTALL_PREFIX=workspace/$CIRCLE_JOB -DCMAKE_CXX_COMPILER_LAUNCHER=ccache'
+          install: true
+      - next-build-template:
+          name: next-macos-xcode11-debug
+          executor_name: macos-11_0_0
+          target_is_macos: true
+          requires:
+            - next-macos-xcode11-release
+          config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=Debug'
+      - next-build-template:
+          name: next-ios-xcode11-release
+          executor_name: macos-11_0_0
+          target_is_macos: true
+          config_params: '-G Ninja -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_SYSTEM_NAME=iOS -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_SYSROOT=iphonesimulator'
+          test_params: '-Q -N'
+  mbgl-legacy:
+    jobs:
+      #
+      # Naming convention: {platform}-{additional description}-{build type}
+      #   - {platform} is the client platform/framework, which may differ from
+      #     the build platform. Specify both if applicable, e.g., "qt5-macos".
+      #   - {additional description} optionally describes the compiler or other
+      #     unique aspect of the build environment.
+      #   - {build type} is typically "debug" or "release".
+      #
+      - android-debug-arm-v7-buck
+      - android-arm-template:
+          name: android-debug-arm-v8
+      - android-arm-template:
+          name: android-gnustl-arm-v7
+          stl: gnustl_shared
+          firebase_device_id: "flo"
+          firebase_device_os: "21"
+          image: android-ndk-r17c:1d5db0eb34
+          abi: arm-v7
+      - android-release:
+          filters:
+            tags:
+              only: /android-v.*/
+      - node-linux-release:
+          filters:
+            tags:
+              only: /node-.*/
+      - node-macos-release:
+          filters:
+            tags:
+              only: /node-.*/
+      - linux-clang-7-sanitize-address-undefined
+      - linux-clang-7-sanitize-thread
+      - linux-gcc5-debug-coverage
+      - linux-doxygen
+      - ios-debug
+      - ios-debug-xcode10
+      - ios-release-template:
+          name: ios-release
+      - ios-release-tag:
+          filters:
+            tags:
+              only: /ios-.*/
+            branches:
+              ignore: /.*/
+      - macos-debug
+  nightly:
+    triggers:
+      - schedule:
+          cron: "0 5 * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - metrics-nightly
+      - ios-release-template:
+          name: ios-release-nightly
+      - ios-sanitize-nightly
+      - ios-sanitize-address-nightly
+      - ios-static-analyzer-nightly
+      - ios-static-analyzer-nightly-xcode10
 
 executors:
   ubuntu-disco:
@@ -707,7 +707,13 @@ jobs:
                 --device-ids flame --os-version-ids 29 --locales en --orientations portrait --timeout 20m \
                 --directories-to-pull /sdcard --results-dir render-test-app-${CIRCLE_BUILD_NUM} \
                 --no-record-video --no-performance-metrics
-              testResult=$(gsutil ls gs://test-lab-186672a0qp5bq-ycr70axads3nc/render-test-app-${CIRCLE_BUILD_NUM}/**/*.html)
+            fi
+      - run:
+          name: Retrieve TestResult from gcloud
+          when: always
+          command: |
+            if [[ -n "${GCLOUD_SERVICE_ACCOUNT_JSON}" && -z "${SKIP_FIREBASE:-}" ]]; then
+              testResult=$(gsutil ls gs://test-lab-186672a0qp5bq-ycr70axads3nc/render-test-app-${CIRCLE_BUILD_NUM}/**/*.html | tail -1)
               mkdir -p /tmp/tests/render-test-app
               gsutil cp $testResult /tmp/tests/render-test-app | true
             fi
diff --git a/next/platform/android/android.cmake b/next/platform/android/android.cmake
index 0f8dd04315..789c30c959 100644
--- a/next/platform/android/android.cmake
+++ b/next/platform/android/android.cmake
@@ -368,7 +368,7 @@ add_custom_command(
         ${CMAKE_COMMAND}
         -E
         tar
-        "chvf"
+        "chf"
         "render-test/android/app/src/main/assets/data.zip"
         --format=zip
         --files-from=render-test/android/app/src/main/assets/to_zip.txt
diff --git a/platform/android/src/test/render_test_runner.cpp b/platform/android/src/test/render_test_runner.cpp
index d0af8d21a6..7e209aff3a 100644
--- a/platform/android/src/test/render_test_runner.cpp
+++ b/platform/android/src/test/render_test_runner.cpp
@@ -79,7 +79,7 @@ std::string jstringToStdString(JNIEnv* env, jstring jStr) {
     return ret;
 }
 
-void changeState(JNIEnv* env, struct android_app* app) {
+void changeState(JNIEnv* env, struct android_app* app, bool result) {
     jobject nativeActivity = app->activity->clazz;
     jclass acl = env->GetObjectClass(nativeActivity);
     jmethodID getClassLoader = env->GetMethodID(acl, "getClassLoader", "()Ljava/lang/ClassLoader;");
@@ -89,6 +89,8 @@ void changeState(JNIEnv* env, struct android_app* app) {
     JavaWrapper<jstring> strClassName(env, env->NewStringUTF("android.app.TestState"));
     jclass testStateClass = static_cast<jclass>(env->CallObjectMethod(cls, findClass, strClassName.get()));
     if (testStateClass != NULL) {
+        jfieldID id0 = env->GetStaticFieldID(testStateClass, "testResult", "Z");
+        env->SetStaticBooleanField(testStateClass, id0, result);
         jfieldID id = env->GetStaticFieldID(testStateClass, "running", "Z");
         env->SetStaticBooleanField(testStateClass, id, false);
     }
@@ -278,10 +280,11 @@ void android_main(struct android_app* app) {
             mbgl::Log::Info(mbgl::Event::General, "End running RenderTestRunner with manifest: '%s'", manifest.c_str());
             return result;
         };
-        runTestWithManifest("/android-manifest-probe-network-gfx.json");
-        runTestWithManifest("/android-manifest-probe-memory.json");
+
+        auto result = runTestWithManifest("/android-manifest-probe-network-gfx.json");
+        result = runTestWithManifest("/android-manifest-probe-memory.json") && result;
         mbgl::Log::Info(mbgl::Event::General, "All tests are finished!");
-        changeState(env, app);
+        changeState(env, app, result);
     }
     while (true) {
         ALooper_pollAll(0, &outFd, &outEvents, reinterpret_cast<void**>(&source));
diff --git a/render-test/android-manifest-probe-memory.json b/render-test/android-manifest-probe-memory.json
index a3562088f4..4a48bc80a9 100644
--- a/render-test/android-manifest-probe-memory.json
+++ b/render-test/android-manifest-probe-memory.json
@@ -1,6 +1,6 @@
 {
     "base_test_path":"mapbox-gl-js/test/integration",
-    "expectation_paths":["render-test/expected"],
+    "expectation_paths":["render-test/expected", "render-test/expected/android"],
     "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
     "vendor_path":"vendor",
     "asset_path": "mapbox-gl-js/test/integration",
diff --git a/render-test/android-manifest-probe-network-gfx.json b/render-test/android-manifest-probe-network-gfx.json
index b16424d999..e8300a036c 100644
--- a/render-test/android-manifest-probe-network-gfx.json
+++ b/render-test/android-manifest-probe-network-gfx.json
@@ -1,6 +1,6 @@
 {
     "base_test_path":"mapbox-gl-js/test/integration",
-    "expectation_paths":["render-test/expected"],
+    "expectation_paths":["render-test/expected", "render-test/expected/android"],
     "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
     "vendor_path":"vendor",
     "asset_path": "mapbox-gl-js/test/integration",
diff --git a/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java b/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java
index 3d8c333902..d508a9d5a4 100644
--- a/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java
+++ b/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java
@@ -6,6 +6,7 @@ import androidx.test.rule.ActivityTestRule;
 
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.Assert;
 import org.junit.runner.RunWith;
 import android.util.Log;
 
@@ -24,6 +25,8 @@ public class NativeActivityTest {
             Log.v("Test", "Test is running");
             Thread.sleep(1000L);
         }
-        Log.v("Test", "End the test");
+        Log.v("Test", "All render tests are finished!");
+        Assert.assertTrue("All test cases are passed", TestState.testResult);
+
     }
 }
\ No newline at end of file
diff --git a/render-test/android/app/src/androidTest/java/android/app/TestState.java b/render-test/android/app/src/androidTest/java/android/app/TestState.java
index 44a0653fb6..b374a72afe 100644
--- a/render-test/android/app/src/androidTest/java/android/app/TestState.java
+++ b/render-test/android/app/src/androidTest/java/android/app/TestState.java
@@ -2,4 +2,5 @@ package android.app;
 
 public class TestState {
     static boolean running = true;
+    static boolean testResult = true;
 }
\ No newline at end of file
diff --git a/render-test/expected/android/render-tests/sprites/1x-screen-1x-icon/expected.png b/render-test/expected/android/render-tests/sprites/1x-screen-1x-icon/expected.png
deleted file mode 100644
index 8a5a9652a7..0000000000
--- a/render-test/expected/android/render-tests/sprites/1x-screen-1x-icon/expected.png
+++ /dev/null
diff --git a/render-test/expected/android/render-tests/sprites/1x-screen-2x-icon/expected.png b/render-test/expected/android/render-tests/sprites/1x-screen-2x-icon/expected.png
deleted file mode 100644
index 235f5170f3..0000000000
--- a/render-test/expected/android/render-tests/sprites/1x-screen-2x-icon/expected.png
+++ /dev/null
diff --git a/render-test/expected/android/render-tests/sprites/2x-screen-1x-icon/expected.png b/render-test/expected/android/render-tests/sprites/2x-screen-1x-icon/expected.png
deleted file mode 100644
index 6fd28edf16..0000000000
--- a/render-test/expected/android/render-tests/sprites/2x-screen-1x-icon/expected.png
+++ /dev/null
diff --git a/render-test/expected/android/render-tests/sprites/2x-screen-2x-icon/expected.png b/render-test/expected/android/render-tests/sprites/2x-screen-2x-icon/expected.png
deleted file mode 100644
index 7ef7af0ab4..0000000000
--- a/render-test/expected/android/render-tests/sprites/2x-screen-2x-icon/expected.png
+++ /dev/null