[render-test] Wrap test data inside RenderTestRunner App + Add a new test app (#15887)

* [render-test] Wrap test resources inside app

* [render-test] Add test app

* fix test app failure

* [render-test]add callback + add javaObjectWrapper

13 files changed, 96 insertions, 59 deletions

diff --git a/render-test/android-manifest.json b/render-test/android-manifest.json
index 56223d4753..7aa9eb26bc 100644
--- a/render-test/android-manifest.json
+++ b/render-test/android-manifest.json
@@ -1,7 +1,8 @@
 {
     "base_test_path":"mapbox-gl-js/test/integration",
-    "expectation_paths":["render-test/expected/render-tests"],
-    "ignore_paths":["platform/node/test/ignores.json", "render-test/linux-ignores.json", "render-test/tests/should-fail.json"],
+    "expectation_paths":[],
+    "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
     "vendor_path":"vendor",
-    "asset_path": "mapbox-gl-js/test/integration"
+    "asset_path": "mapbox-gl-js/test/integration",
+    "result_path": "/sdcard/"
 }
\ No newline at end of file
diff --git a/render-test/android/README.md b/render-test/android/README.md
index 270c970fce..4037419012 100644
--- a/render-test/android/README.md
+++ b/render-test/android/README.md
@@ -1,9 +1,3 @@
 # RenderTestRunner
 
-This app is a purely native application, with no Java source code, that can run **mbgl-render-test-runner** on android devices.
-
-
-## Setup the test environment
-- Run render_test_setup.sh so that all the necessary test resources are pushed to the device.
-
-- Switch on storage permission of the app so that it can read/write data on SD card.
\ No newline at end of file
+This app is a purely native application, with no Java source code, that can run **mbgl-render-test-runner** on android devices.
\ No newline at end of file
diff --git a/render-test/android/app/build.gradle b/render-test/android/app/build.gradle
index 60609e3ba2..682af85dcf 100644
--- a/render-test/android/app/build.gradle
+++ b/render-test/android/app/build.gradle
@@ -14,6 +14,7 @@ android {
                 targets 'mbgl-render-test-runner'
             }
         }
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
     }
     externalNativeBuild {
         cmake {
@@ -26,4 +27,7 @@ android {
 dependencies {
     implementation 'androidx.appcompat:appcompat:1.0.2'
     implementation 'androidx.constraintlayout:constraintlayout:1.1.3'
+    androidTestImplementation 'androidx.test.ext:junit:1.1.0'
+    androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0-beta01'
+    androidTestImplementation 'androidx.test:rules:1.2.0-beta01'
 }
diff --git a/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java b/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java
new file mode 100644
index 0000000000..3d8c333902
--- /dev/null
+++ b/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java
@@ -0,0 +1,29 @@
+package android.app;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import androidx.test.filters.LargeTest;
+import androidx.test.rule.ActivityTestRule;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import android.util.Log;
+
+@LargeTest
+@RunWith(AndroidJUnit4.class)
+public class NativeActivityTest {
+
+    @Rule
+    public ActivityTestRule<NativeActivity> mActivityTestRule = new ActivityTestRule<>(NativeActivity.class, false, false);
+
+    @Test(timeout = 1200000L)
+    public void runRenderTests() throws Exception {
+        Log.v("Test", "Start the test");
+        mActivityTestRule.launchActivity(null);
+        while (TestState.running) {
+            Log.v("Test", "Test is running");
+            Thread.sleep(1000L);
+        }
+        Log.v("Test", "End the test");
+    }
+}
\ No newline at end of file
diff --git a/render-test/android/app/src/androidTest/java/android/app/TestState.java b/render-test/android/app/src/androidTest/java/android/app/TestState.java
new file mode 100644
index 0000000000..44a0653fb6
--- /dev/null
+++ b/render-test/android/app/src/androidTest/java/android/app/TestState.java
@@ -0,0 +1,5 @@
+package android.app;
+
+public class TestState {
+    static boolean running = true;
+}
\ No newline at end of file
diff --git a/render-test/android/app/src/main/AndroidManifest.xml b/render-test/android/app/src/main/AndroidManifest.xml
index 6c7af7ed8f..8df48ef97f 100644
--- a/render-test/android/app/src/main/AndroidManifest.xml
+++ b/render-test/android/app/src/main/AndroidManifest.xml
@@ -9,7 +9,7 @@
       android:fullBackupContent="false"
       android:icon="@mipmap/ic_launcher"
       android:label="@string/app_name"
-      android:hasCode="false">
+      android:hasCode="true">
 
     <activity android:name="android.app.NativeActivity"
               android:label="@string/app_name"
diff --git a/render-test/android/app/src/main/assets/to_zip.txt b/render-test/android/app/src/main/assets/to_zip.txt
new file mode 100644
index 0000000000..7023ad9cfa
--- /dev/null
+++ b/render-test/android/app/src/main/assets/to_zip.txt
@@ -0,0 +1,15 @@
+mapbox-gl-js/test/integration/data/
+mapbox-gl-js/test/integration/video/
+mapbox-gl-js/test/integration/tilesets/
+mapbox-gl-js/test/integration/tiles/
+mapbox-gl-js/test/integration/styles/
+mapbox-gl-js/test/integration/render-tests/
+mapbox-gl-js/test/integration/query-tests/
+mapbox-gl-js/test/integration/image/
+mapbox-gl-js/test/integration/glyphs/
+mapbox-gl-js/test/integration/geojson/
+mapbox-gl-js/test/integration/sprites/
+vendor/mapbox-gl-styles/styles/
+vendor/mapbox-gl-styles/sprites/
+ignores/
+android-manifest.json
\ No newline at end of file
diff --git a/render-test/android/render_test_setup.sh b/render-test/android/render_test_setup.sh
deleted file mode 100755
index 1dea44399e..0000000000
--- a/render-test/android/render_test_setup.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-adb shell rm -rf /sdcard/render-test
-adb shell mkdir /sdcard/render-test
-adb shell mkdir /sdcard/render-test/vendor
-adb shell mkdir /sdcard/render-test/ignores
-adb shell mkdir /sdcard/render-test/render-test/tests
-
-# push test sources
-adb push ../../mapbox-gl-js/test/integration/render-tests /sdcard/render-test/mapbox-gl-js/test/integration/render-tests
-adb push ../../mapbox-gl-js/test/integration/query-tests /sdcard/render-test/mapbox-gl-js/test/integration/query-tests 
-adb push ../../mapbox-gl-js/test/integration/tiles /sdcard/render-test/mapbox-gl-js/test/integration/tiles
-adb push ../../mapbox-gl-js/test/integration/glyphs /sdcard/render-test/mapbox-gl-js/test/integration/glyphs
-adb push ../../mapbox-gl-js/test/integration/styles /sdcard/render-test/mapbox-gl-js/test/integration/styles 
-adb push ../../mapbox-gl-js/test/integration/tilesets /sdcard/render-test/mapbox-gl-js/test/integration/tilesets 
-adb push ../../mapbox-gl-js/test/integration/image /sdcard/render-test/mapbox-gl-js/test/integration/image 
-adb push ../../mapbox-gl-js/test/integration/video /sdcard/render-test/mapbox-gl-js/test/integration/video 
-adb push ../../vendor/mapbox-gl-styles/styles /sdcard/render-test/vendor/mapbox-gl-styles/styles 
-adb push ../../vendor/mapbox-gl-styles/sprites /sdcard/render-test/vendor/mapbox-gl-styles/sprites 
-adb push ../../mapbox-gl-js/test/integration/data /sdcard/render-test/mapbox-gl-js/test/integration/data 
-adb push ../../mapbox-gl-js/test/integration/geojson /sdcard/render-test/mapbox-gl-js/test/integration/geojson
-mkdir sprites
-cp -r ../../mapbox-gl-js/test/integration/sprites/ sprites
-adb push sprites /sdcard/render-test/mapbox-gl-js/test/integration/sprites
-rm -rf sprites
-
-# push extra expectations
-adb push ../../render-test/expected/render-tests /sdcard/render-test/render-test/expected/render-tests
-
-# push default ignore lists
-adb shell mkdir /sdcard/render-test/platform
-adb shell mkdir /sdcard/render-test/platform/node
-adb shell mkdir /sdcard/render-test/platform/node/test
-adb push ../../platform/node/test/ignores.json /sdcard/render-test/platform/node/test
-adb shell mkdir /sdcard/render-test/render-test
-adb push ../linux-ignores.json /sdcard/render-test/render-test
-adb push ../tests/should-fail.json /sdcard/render-test/render-test/tests
-
-# push manifest
-adb push ../android-manifest.json /sdcard/render-test
-
-adb shell ls /sdcard/render-test/
\ No newline at end of file
diff --git a/render-test/include/mbgl/render_test.hpp b/render-test/include/mbgl/render_test.hpp
index 8a82079bee..1669e59932 100644
--- a/render-test/include/mbgl/render_test.hpp
+++ b/render-test/include/mbgl/render_test.hpp
@@ -1,6 +1,9 @@
 #pragma once
+
+#include <functional>
+
 namespace mbgl {
 
-int runRenderTests(int argc, char* argv[]);
+int runRenderTests(int argc, char* argv[], std::function<void()>);
 
 } // namespace mbgl
diff --git a/render-test/manifest_parser.cpp b/render-test/manifest_parser.cpp
index 3f1f9f3866..6bba33754a 100644
--- a/render-test/manifest_parser.cpp
+++ b/render-test/manifest_parser.cpp
@@ -34,9 +34,15 @@ const std::vector<std::pair<std::string, std::string>>& Manifest::getIgnores() c
 const std::string& Manifest::getTestRootPath() const {
     return testRootPath;
 }
+const std::string& Manifest::getAssetPath() const {
+    return assetPath;
+}
 const std::string& Manifest::getManifestPath() const {
     return manifestPath;
 }
+const std::string& Manifest::getResultPath() const {
+    return resultPath;
+}
 
 void Manifest::doShuffle(uint32_t seed) {
     std::seed_seq sequence{seed};
@@ -239,7 +245,7 @@ mbgl::filesystem::path getValidPath(const std::string& manifestPath, const std::
         result = BasePath / result;
     }
     if (mbgl::filesystem::exists(result)) {
-        return result;
+        return result.lexically_normal();
     }
     mbgl::Log::Warning(mbgl::Event::General, "Invalid path is provoided inside the manifest file: %s", path.c_str());
     return mbgl::filesystem::path{};
@@ -285,6 +291,18 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manife
             return mbgl::nullopt;
         }
     }
+    if (document.HasMember("result_path")) {
+        const auto& resultPathValue = document["result_path"];
+        if (!resultPathValue.IsString()) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Invalid assetPath is provoided inside the manifest file: %s", filePath.c_str());
+            return mbgl::nullopt;
+        }
+        manifest.resultPath = (getValidPath(manifest.manifestPath, resultPathValue.GetString()) / "").string();
+        if (manifest.resultPath.empty()) {
+            return mbgl::nullopt;
+        }
+    }
     mbgl::filesystem::path baseTestPath;
     if (document.HasMember("base_test_path")) {
         const auto& testPathValue = document["base_test_path"];
@@ -366,6 +384,11 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manife
     if (manifest.manifestPath.back() == '/') {
         manifest.manifestPath.pop_back();
     }
+    if (manifest.resultPath.empty()) {
+        manifest.resultPath = manifest.manifestPath;
+    } else if (manifest.resultPath.back() == '/') {
+        manifest.resultPath.pop_back();
+    }
 
     std::vector<mbgl::filesystem::path> paths;
     for (const auto& id : testNames) {
diff --git a/render-test/manifest_parser.hpp b/render-test/manifest_parser.hpp
index bc5adf1091..c4672fb4c5 100644
--- a/render-test/manifest_parser.hpp
+++ b/render-test/manifest_parser.hpp
@@ -17,7 +17,9 @@ public:
     const std::vector<std::pair<std::string, std::string>>& getIgnores() const;
     const std::vector<TestPaths>& getTestPaths() const;
     const std::string& getTestRootPath() const;
+    const std::string& getAssetPath() const;
     const std::string& getManifestPath() const;
+    const std::string& getResultPath() const;
     void doShuffle(uint32_t seed);
 
     std::string localizeURL(const std::string& url) const;
@@ -43,6 +45,7 @@ private:
     std::string testRootPath;
     std::string vendorPath;
     std::string assetPath;
+    std::string resultPath;
     std::vector<std::pair<std::string, std::string>> ignores;
     std::vector<TestPaths> testPaths;
 };
diff --git a/render-test/render_test.cpp b/render-test/render_test.cpp
index 38d6c15f3f..5753065a7e 100644
--- a/render-test/render_test.cpp
+++ b/render-test/render_test.cpp
@@ -100,7 +100,7 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
 } // namespace
 namespace mbgl {
 
-int runRenderTests(int argc, char** argv) {
+int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
     bool recycleMap;
     bool shuffle;
     uint32_t seed;
@@ -198,10 +198,12 @@ int runRenderTests(int argc, char** argv) {
         }
 
         metadatas.push_back(std::move(metadata));
+        if (testStatus) {
+            testStatus();
+        }
     }
-    const auto& testRootPath = manifest.getManifestPath();
     const auto resultPath =
-        testRootPath + "/" + (testNames.empty() ? "render-tests" : testNames.front()) + "_index.html";
+        manifest.getResultPath() + "/" + (testNames.empty() ? "render-tests" : testNames.front()) + "_index.html";
     std::string resultsHTML = createResultPage(stats, metadatas, shuffle, seed);
     mbgl::util::write_file(resultPath, resultsHTML);
 
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 0389b83575..ce76bda157 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -596,7 +596,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata, R
         std::string imagePath = operationArray[2].GetString();
         imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end());
 
-        const mbgl::filesystem::path filePath = mbgl::filesystem::path(manifest.getTestRootPath()) / imagePath;
+        const mbgl::filesystem::path filePath = (mbgl::filesystem::path(manifest.getAssetPath()) / imagePath);
 
         mbgl::optional<std::string> maybeImage = mbgl::util::readFile(filePath.string());
         if (!maybeImage) {