Make renderTestrunner run two times with different manifest configupstream/zmiao-android-baseline

7 files changed, 56 insertions, 21 deletions

diff --git a/.gitignore b/.gitignore
index 33d3d987d8..d81b8c9727 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,6 +27,7 @@ xcuserdata
 /platform/ios/benchmark/assets/glyphs/DIN*
 /platform/ios/benchmark/assets/tiles/mapbox.mapbox-terrain-v2,mapbox.mapbox-streets-v6
 /platform/darwin/developer.xcconfig
+**/token
 /platform/macos/macos.xcworkspace/xcshareddata/macos.xcscmblueprint
 /platform/ios/ios.xcworkspace/xcshareddata/ios.xcscmblueprint
 /documentation
diff --git a/circle.yml b/circle.yml
index 447b98f0a4..27cdf7c0ea 100644
--- a/circle.yml
+++ b/circle.yml
@@ -707,7 +707,7 @@ jobs:
                 --device-ids flame --os-version-ids 29 --locales en --orientations portrait --timeout 20m \
                 --directories-to-pull /sdcard --results-dir render-test-app-${CIRCLE_BUILD_NUM} \
                 --no-record-video --no-performance-metrics
-              testResult=$(gsutil ls gs://test-lab-186672a0qp5bq-ycr70axads3nc/render-test-app-${CIRCLE_BUILD_NUM}/**/*.html | tail -1)
+              testResult=$(gsutil ls gs://test-lab-186672a0qp5bq-ycr70axads3nc/render-test-app-${CIRCLE_BUILD_NUM}/**/*.html)
               mkdir -p /tmp/tests/render-test-app
               gsutil cp $testResult /tmp/tests/render-test-app | true
             fi
diff --git a/next/platform/android/android.cmake b/next/platform/android/android.cmake
index b0c9c712dc..befdc6f31a 100644
--- a/next/platform/android/android.cmake
+++ b/next/platform/android/android.cmake
@@ -344,8 +344,14 @@ add_custom_command(
         ${CMAKE_COMMAND}
         -E
         copy
-        ${MBGL_ROOT}/render-test/android-manifest.json
-        ${MBGL_ROOT}/android-manifest.json
+        ${MBGL_ROOT}/render-test/android-manifest-probe-memory.json
+        ${MBGL_ROOT}/android-manifest-probe-memory.json
+    COMMAND
+        ${CMAKE_COMMAND}
+        -E
+        copy
+        ${MBGL_ROOT}/render-test/android-manifest-probe-network-gfx.json
+        ${MBGL_ROOT}/android-manifest-probe-network-gfx.json
     COMMAND
         ${CMAKE_COMMAND}
         -E
@@ -362,7 +368,7 @@ add_custom_command(
         ${CMAKE_COMMAND}
         -E
         tar
-        "cf"
+        "chvf"
         "render-test/android/app/src/main/assets/data.zip"
         --format=zip
         --files-from=render-test/android/app/src/main/assets/to_zip.txt
diff --git a/platform/android/src/test/render_test_runner.cpp b/platform/android/src/test/render_test_runner.cpp
index 98197ce1cc..cef89e9448 100644
--- a/platform/android/src/test/render_test_runner.cpp
+++ b/platform/android/src/test/render_test_runner.cpp
@@ -252,26 +252,33 @@ void android_main(struct android_app* app) {
     } else {
         unZipFile(env, zipFile, storagePath);
 
-        std::string configFile = storagePath + "/android-manifest.json";
-        std::vector<std::string> arguments = {"mbgl-render-test-runner", "-p", configFile};
-        std::vector<char*> argv;
-        for (const auto& arg : arguments) {
-            argv.push_back((char*)arg.data());
-        }
-        argv.push_back(nullptr);
+        auto runTestWithManifest = [&](const std::string manifest, const std::string filter = {}) -> bool {
+            std::string configFile = storagePath + manifest;
+            std::vector<std::string> arguments = {"mbgl-render-test-runner", "-p", configFile, filter};
+            std::vector<char*> argv;
+            for (const auto& arg : arguments) {
+                argv.push_back((char*)arg.data());
+            }
+            argv.push_back(nullptr);
 
-        int finishedTestCount = 0;
-        std::function<void()> testStatus = [&]() {
-            ALooper_pollAll(0, &outFd, &outEvents, reinterpret_cast<void**>(&source));
+            int finishedTestCount = 0;
+            std::function<void()> testStatus = [&]() {
+                ALooper_pollAll(0, &outFd, &outEvents, reinterpret_cast<void**>(&source));
 
-            if (source != nullptr) {
-                source->process(app, source);
-            }
+                if (source != nullptr) {
+                    source->process(app, source);
+                }
 
-            mbgl::Log::Info(mbgl::Event::General, "Current finished tests number is '%d' ", ++finishedTestCount);
+                mbgl::Log::Info(mbgl::Event::General, "Current finished tests number is '%d' ", ++finishedTestCount);
+            };
+            mbgl::Log::Info(
+                mbgl::Event::General, "Start running RenderTestRunner with manifest: '%s'", manifest.c_str());
+            bool result = mbgl::runRenderTests(argv.size() - 1, argv.data(), testStatus) == 0;
+            mbgl::Log::Info(mbgl::Event::General, "End running RenderTestRunner with manifest: '%s'", manifest.c_str());
+            return result;
         };
-
-        mbgl::runRenderTests(argv.size() - 1, argv.data(), testStatus);
+        runTestWithManifest("/android-manifest-probe-network-gfx.json");
+        runTestWithManifest("/android-manifest-probe-memory.json", "--filter=real-world");
         mbgl::Log::Info(mbgl::Event::General, "All tests are finished!");
         changeState(env, app);
     }
diff --git a/render-test/android-manifest-probe-memory.json b/render-test/android-manifest-probe-memory.json
new file mode 100644
index 0000000000..bc94465761
--- /dev/null
+++ b/render-test/android-manifest-probe-memory.json
@@ -0,0 +1,10 @@
+{
+    "base_test_path":"mapbox-gl-js/test/integration",
+    "expectation_paths":["render-test/expected"],
+    "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
+    "vendor_path":"vendor",
+    "asset_path": "mapbox-gl-js/test/integration",
+    "result_path":"/sdcard/",
+    "probes": ["probeMemory"],
+    "metric_path":"render-test/metrics/android"
+}  
\ No newline at end of file
diff --git a/render-test/android-manifest-probe-network-gfx.json b/render-test/android-manifest-probe-network-gfx.json
new file mode 100644
index 0000000000..b16424d999
--- /dev/null
+++ b/render-test/android-manifest-probe-network-gfx.json
@@ -0,0 +1,10 @@
+{
+    "base_test_path":"mapbox-gl-js/test/integration",
+    "expectation_paths":["render-test/expected"],
+    "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
+    "vendor_path":"vendor",
+    "asset_path": "mapbox-gl-js/test/integration",
+    "result_path":"/sdcard/",
+    "probes": ["probeGFX", "probeNetwork"],
+    "metric_path":"render-test/metrics/android"
+}  
\ No newline at end of file
diff --git a/render-test/android/app/src/main/assets/to_zip.txt b/render-test/android/app/src/main/assets/to_zip.txt
index aed98772c9..fbf9539583 100644
--- a/render-test/android/app/src/main/assets/to_zip.txt
+++ b/render-test/android/app/src/main/assets/to_zip.txt
@@ -15,4 +15,5 @@ vendor/mvt-fixtures/real-world/
 ignores/
 render-test/metrics/android/
 render-test/expected/
-android-manifest.json
\ No newline at end of file
+android-manifest-probe-network-gfx.json
+android-manifest-probe-memory.json
\ No newline at end of file