[render-test] Add test appupstream/zmiao-independent-test-runner

15 files changed, 196 insertions, 131 deletions

diff --git a/next/platform/android/android.cmake b/next/platform/android/android.cmake
index 06183291db..e086f8b75b 100644
--- a/next/platform/android/android.cmake
+++ b/next/platform/android/android.cmake
@@ -338,6 +338,47 @@ target_link_libraries(
         mbgl-render-test
 )
 
+add_custom_command(
+    TARGET mbgl-render-test-runner PRE_BUILD
+    COMMAND
+        ${CMAKE_COMMAND}
+        -E
+        copy
+        ${MBGL_ROOT}/render-test/android-manifest.json
+        ${MBGL_ROOT}/android-manifest.json
+    COMMAND
+        ${CMAKE_COMMAND}
+        -E
+        copy
+        ${MBGL_ROOT}/platform/node/test/ignores.json
+        ${MBGL_ROOT}/ignores/ignores.json
+    COMMAND
+        ${CMAKE_COMMAND}
+        -E
+        copy
+        ${MBGL_ROOT}/render-test/linux-ignores.json
+        ${MBGL_ROOT}/ignores/linux-ignores.json
+    COMMAND
+        ${CMAKE_COMMAND}
+        -E
+        tar
+        "cf"
+        "render-test/android/app/src/main/assets/data.zip"
+        --format=zip
+        --files-from=render-test/android/app/src/main/assets/to_zip.txt
+    COMMAND
+        ${CMAKE_COMMAND}
+        -E
+        remove_directory
+        ${MBGL_ROOT}/ignores
+    COMMAND
+        ${CMAKE_COMMAND}
+        -E
+        remove
+        ${MBGL_ROOT}/android-manifest.json
+    WORKING_DIRECTORY ${MBGL_ROOT}
+)
+
 # Android has no concept of MinSizeRel on android.toolchain.cmake and provides configurations tuned for binary size. We can push it a bit
 # more with code folding and LTO.
 set_target_properties(example-custom-layer PROPERTIES LINK_FLAGS_RELEASE "-fuse-ld=gold -O2 -flto -Wl,--icf=safe")
diff --git a/next/render-test/CMakeLists.txt b/next/render-test/CMakeLists.txt
index 0fef727c34..fa4bb0d615 100644
--- a/next/render-test/CMakeLists.txt
+++ b/next/render-test/CMakeLists.txt
@@ -38,8 +38,12 @@ include(${PROJECT_SOURCE_DIR}/vendor/boost.cmake)
 
 target_link_libraries(
     mbgl-render-test
-    PRIVATE Mapbox::Base::Extras::args Mapbox::Base::pixelmatch-cpp
-    PUBLIC mbgl-core mbgl-vendor-boost Mapbox::Base::Extras::filesystem
+    PRIVATE
+        Mapbox::Base::Extras::args
+        Mapbox::Base::Extras::filesystem
+        Mapbox::Base::pixelmatch-cpp
+        mbgl-vendor-boost
+    PUBLIC mbgl-core
 )
 
 if(CMAKE_SYSTEM_NAME STREQUAL Android)
diff --git a/platform/android/src/test/render_test_runner.cpp b/platform/android/src/test/render_test_runner.cpp
index 83e0a11c91..7dae0fc448 100644
--- a/platform/android/src/test/render_test_runner.cpp
+++ b/platform/android/src/test/render_test_runner.cpp
@@ -1,5 +1,4 @@
 #include <android_native_app_glue.h>
-#include <ghc/filesystem.hpp>
 #include <mbgl/render_test.hpp>
 #include <mbgl/util/logging.hpp>
 
@@ -8,6 +7,7 @@
 
 #include "jni.hpp"
 
+#include <cstdio>
 #include <string>
 #include <vector>
 
@@ -42,34 +42,30 @@ void Log::platformRecord(EventSeverity severity, const std::string& msg) {
 
 } // namespace mbgl
 
-void copyFile(AAssetManager* assetManager, const std::string& filePath, const std::string& fileName) {
-    if (ghc::filesystem::exists(filePath)) {
-        mbgl::Log::Warning(mbgl::Event::General, "File '%s' already exists", filePath.c_str());
+bool copyFile(AAssetManager* assetManager, const std::string& filePath, const std::string& fileName) {
+    AAsset* fileAsset = AAssetManager_open(assetManager, fileName.c_str(), AASSET_MODE_BUFFER);
+    const void* fileData = AAsset_getBuffer(fileAsset);
+    const off_t fileLen = AAsset_getLength(fileAsset);
+
+    FILE* newFile = std::fopen(filePath.c_str(), "w+");
+    bool stateOk = newFile != NULL;
+    if (!stateOk) {
+        mbgl::Log::Warning(mbgl::Event::General, "Failed to create new file entry %s", fileName.c_str());
     } else {
-        AAsset* fileAsset = AAssetManager_open(assetManager, fileName.c_str(), AASSET_MODE_BUFFER);
-        const void* fileData = AAsset_getBuffer(fileAsset);
-        const off_t fileLen = AAsset_getLength(fileAsset);
-
-        FILE* newFile = std::fopen(filePath.c_str(), "w+");
-        if (NULL == newFile) {
-            mbgl::Log::Warning(mbgl::Event::General, "Failed to create new file entry %s", fileName.c_str());
-        } else {
-            auto res = std::fwrite(fileData, sizeof(char), fileLen, newFile);
-            if (fileLen != res) {
-                mbgl::Log::Warning(
-                    mbgl::Event::General, "Failed to generate file entry %s from assets", fileName.c_str());
-            }
+        auto res = std::fwrite(fileData, sizeof(char), fileLen, newFile);
+        if (fileLen != res) {
+            mbgl::Log::Warning(mbgl::Event::General, "Failed to generate file entry %s from assets", fileName.c_str());
         }
-        std::fclose(newFile);
-        AAsset_close(fileAsset);
     }
+    std::fclose(newFile);
+    AAsset_close(fileAsset);
+    return stateOk;
 }
 
 std::string jstring2string(JNIEnv* env, jstring jStr) {
     if (!jStr) {
         return "";
     }
-
     const jclass stringClass = env->GetObjectClass(jStr);
     const jmethodID getBytes = env->GetMethodID(stringClass, "getBytes", "(Ljava/lang/String;)[B");
     const jbyteArray stringJbytes = (jbyteArray)env->CallObjectMethod(jStr, getBytes, env->NewStringUTF("UTF-8"));
@@ -91,11 +87,28 @@ void android_main(struct android_app* app) {
     app->activity->vm->AttachCurrentThread(&env, NULL);
     const char* storage_chars = app->activity->internalDataPath;
     std::string storagePath(storage_chars);
+    std::string zipFile = storagePath + "/data.zip";
 
-    std::string zipFile = storagePath + "/res.zip";
+    jclass fileClass = env->FindClass("java/io/File");
+    jmethodID fileCtor = env->GetMethodID(fileClass, "<init>", "(Ljava/lang/String;Ljava/lang/String;)V");
+    jmethodID fileExists = env->GetMethodID(fileClass, "exists", "()Z");
 
-    AAssetManager* assetManager = app->activity->assetManager;
-    copyFile(assetManager, zipFile, "res.zip");
+    jstring destination = env->NewStringUTF(storage_chars);
+    jstring fileName = env->NewStringUTF((std::string("data.zip")).c_str());
+    jobject fileToCopy = env->NewObject(fileClass, fileCtor, destination, fileName);
+    if (env->CallBooleanMethod(fileToCopy, fileExists)) {
+        mbgl::Log::Warning(mbgl::Event::General, "File '%s' already exists", zipFile.c_str());
+    } else if (!copyFile(app->activity->assetManager, zipFile, "data.zip")) {
+        mbgl::Log::Error(
+            mbgl::Event::General, "Failed to copy zip File '%s' to external storage for upzipping", zipFile.c_str());
+        env->DeleteLocalRef(fileClass);
+        return;
+    }
+
+    jmethodID fileIsDirectory = env->GetMethodID(fileClass, "isDirectory", "()Z");
+    jmethodID deleteFile = env->GetMethodID(fileClass, "delete", "()Z");
+    jmethodID createNewFile = env->GetMethodID(fileClass, "createNewFile", "()Z");
+    jmethodID fileGetName = env->GetMethodID(fileClass, "getName", "()Ljava/lang/String;");
 
     jclass fileInputStream = env->FindClass("java/io/FileInputStream");
     jmethodID finCtor = env->GetMethodID(fileInputStream, "<init>", "(Ljava/lang/String;)V");
@@ -112,60 +125,63 @@ void android_main(struct android_app* app) {
     jmethodID zinCloseEntry = env->GetMethodID(zipInputStream, "closeEntry", "()V");
 
     jclass zipEntry = env->FindClass("java/util/zip/ZipEntry");
-    jmethodID getName = env->GetMethodID(zipEntry, "getName", "()Ljava/lang/String;");
+    jmethodID zipGetName = env->GetMethodID(zipEntry, "getName", "()Ljava/lang/String;");
     jmethodID zipIsDirectory = env->GetMethodID(zipEntry, "isDirectory", "()Z");
 
-    jclass fileClass = env->FindClass("java/io/File");
-    jmethodID fileCtor = env->GetMethodID(fileClass, "<init>", "(Ljava/lang/String;Ljava/lang/String;)V");
-    jmethodID fileIsDirectory = env->GetMethodID(fileClass, "isDirectory", "()Z");
-    jmethodID exists = env->GetMethodID(fileClass, "exists", "()Z");
-    jmethodID createNewFile = env->GetMethodID(fileClass, "createNewFile", "()Z");
-    jmethodID fileGetName = env->GetMethodID(fileClass, "getName", "()Ljava/lang/String;");
-
     // Upzip the resource folder to destination path
-    jstring destination = env->NewStringUTF(storage_chars);
-    jstring jstr = env->NewStringUTF(zipFile.c_str());
-    jobject fin = env->NewObject(fileInputStream, finCtor, jstr);
-    jobject zin = env->NewObject(zipInputStream, zinCtor, fin);
-    jobject ze = NULL;
-    while ((ze = env->CallObjectMethod(zin, zinGetNextEntry)) != NULL) {
-        jstring dir = (jstring)env->CallObjectMethod(ze, getName);
+    jstring jStr = env->NewStringUTF(zipFile.c_str());
+    jobject fileIn = env->NewObject(fileInputStream, finCtor, jStr);
+    jobject zipIn = env->NewObject(zipInputStream, zinCtor, fileIn);
+    jobject zEntry = NULL;
+    while ((zEntry = env->CallObjectMethod(zipIn, zinGetNextEntry)) != NULL) {
+        jstring dir = (jstring)env->CallObjectMethod(zEntry, zipGetName);
         std::string name = jstring2string(env, dir);
-        bool ok = env->CallBooleanMethod(ze, zipIsDirectory);
+        bool isDir = env->CallBooleanMethod(zEntry, zipIsDirectory);
 
         jobject f = env->NewObject(fileClass, fileCtor, destination, dir);
-        if (ok) {
+        if (isDir) {
             if (!(env->CallBooleanMethod(f, fileIsDirectory))) {
                 jmethodID mkdirs = env->GetMethodID(fileClass, "mkdirs", "()Z");
-                jboolean success = (env->CallBooleanMethod(f, mkdirs));
+                bool success = (env->CallBooleanMethod(f, mkdirs));
                 std::string fileName = jstring2string(env, (jstring)env->CallObjectMethod(f, fileGetName));
                 if (!success) {
                     mbgl::Log::Warning(
                         mbgl::Event::General, "Failed to create folder entry %s from zip", fileName.c_str());
                 }
             }
-        } else {
-            if (!(env->CallBooleanMethod(f, exists))) {
-                jboolean success = env->CallBooleanMethod(f, createNewFile);
-                std::string fileName = jstring2string(env, (jstring)env->CallObjectMethod(f, fileGetName));
-                if (!success) {
-                    mbgl::Log::Warning(
-                        mbgl::Event::General, "Failed to create folder entry %s from zip", fileName.c_str());
-                    continue;
-                }
-                jobject fout = env->NewObject(fileOutputStream, foutCtor, f);
-                jbyteArray jBuff = env->NewByteArray(2048);
-                int count;
-                while ((count = env->CallIntMethod(zin, zinRead, jBuff)) != -1) {
-                    env->CallVoidMethod(fout, foutWrite, jBuff, 0, count);
-                }
-                env->CallVoidMethod(zin, zinCloseEntry);
-                env->CallVoidMethod(fout, foutClose);
-                env->DeleteLocalRef(jBuff);
+        } else if (!(env->CallBooleanMethod(f, fileExists))) {
+            bool success = env->CallBooleanMethod(f, createNewFile);
+            std::string fileName = jstring2string(env, (jstring)env->CallObjectMethod(f, fileGetName));
+            if (!success) {
+                mbgl::Log::Warning(mbgl::Event::General, "Failed to create folder entry %s from zip", fileName.c_str());
+                continue;
+            }
+            jobject fout = env->NewObject(fileOutputStream, foutCtor, f);
+            jbyteArray jBuff = env->NewByteArray(2048);
+            int count;
+            while ((count = env->CallIntMethod(zipIn, zinRead, jBuff)) != -1) {
+                env->CallVoidMethod(fout, foutWrite, jBuff, 0, count);
             }
+            env->CallVoidMethod(zipIn, zinCloseEntry);
+            env->CallVoidMethod(fout, foutClose);
+            env->DeleteLocalRef(jBuff);
         }
     }
 
+    jobject fileToDelete = env->NewObject(fileClass, fileCtor, destination, fileName);
+    if (env->CallBooleanMethod(fileToDelete, fileExists)) {
+        jboolean success = (env->CallBooleanMethod(fileToDelete, deleteFile));
+        if (!success) {
+            mbgl::Log::Warning(mbgl::Event::General, "Failed to delete file entry %s", zipFile.c_str());
+        }
+    }
+
+    env->DeleteLocalRef(fileInputStream);
+    env->DeleteLocalRef(fileOutputStream);
+    env->DeleteLocalRef(zipInputStream);
+    env->DeleteLocalRef(zipEntry);
+    env->DeleteLocalRef(fileClass);
+
     std::string configFile = storagePath + "/android-manifest.json";
 
     std::vector<std::string> arguments = {"mbgl-render-test-runner", "-p", configFile};
@@ -175,6 +191,5 @@ void android_main(struct android_app* app) {
     }
     argv.push_back(nullptr);
     (void)mbgl::runRenderTests(argv.size() - 1, argv.data());
-
     app->activity->vm->DetachCurrentThread();
 }
\ No newline at end of file
diff --git a/render-test/android-manifest.json b/render-test/android-manifest.json
index 9cd9d62a7f..7aa9eb26bc 100644
--- a/render-test/android-manifest.json
+++ b/render-test/android-manifest.json
@@ -1,7 +1,8 @@
 {
-    "base_test_path":".",
+    "base_test_path":"mapbox-gl-js/test/integration",
     "expectation_paths":[],
     "ignore_paths":["ignores/ignores.json", "ignores/linux-ignores.json"],
     "vendor_path":"vendor",
-    "asset_path": "."
+    "asset_path": "mapbox-gl-js/test/integration",
+    "result_path": "/sdcard/"
 }
\ No newline at end of file
diff --git a/render-test/android/README.md b/render-test/android/README.md
index 270c970fce..4037419012 100644
--- a/render-test/android/README.md
+++ b/render-test/android/README.md
@@ -1,9 +1,3 @@
 # RenderTestRunner
 
-This app is a purely native application, with no Java source code, that can run **mbgl-render-test-runner** on android devices.
-
-
-## Setup the test environment
-- Run render_test_setup.sh so that all the necessary test resources are pushed to the device.
-
-- Switch on storage permission of the app so that it can read/write data on SD card.
\ No newline at end of file
+This app is a purely native application, with no Java source code, that can run **mbgl-render-test-runner** on android devices.
\ No newline at end of file
diff --git a/render-test/android/app/build.gradle b/render-test/android/app/build.gradle
index d5115e8aa3..682af85dcf 100644
--- a/render-test/android/app/build.gradle
+++ b/render-test/android/app/build.gradle
@@ -14,11 +14,7 @@ android {
                 targets 'mbgl-render-test-runner'
             }
         }
-        ndk {
-            // Tells Gradle to build outputs for the following ABIs and package
-            // them into your APK.
-            abiFilters 'arm64-v8a'
-        }
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
     }
     externalNativeBuild {
         cmake {
@@ -31,6 +27,7 @@ android {
 dependencies {
     implementation 'androidx.appcompat:appcompat:1.0.2'
     implementation 'androidx.constraintlayout:constraintlayout:1.1.3'
+    androidTestImplementation 'androidx.test.ext:junit:1.1.0'
     androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0-beta01'
     androidTestImplementation 'androidx.test:rules:1.2.0-beta01'
 }
diff --git a/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java b/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java
new file mode 100644
index 0000000000..e52c0228f4
--- /dev/null
+++ b/render-test/android/app/src/androidTest/java/android/app/NativeActivityTest.java
@@ -0,0 +1,23 @@
+package android.app;
+
+
+import androidx.test.filters.LargeTest;
+import androidx.test.rule.ActivityTestRule;
+import androidx.test.runner.AndroidJUnit4;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+@LargeTest
+@RunWith(AndroidJUnit4.class)
+public class NativeActivityTest {
+
+    @Rule
+    public ActivityTestRule<NativeActivity> mActivityTestRule = new ActivityTestRule<>(NativeActivity.class);
+
+    @Test
+    public void nativeActivityTest() {
+        mActivityTestRule.launchActivity(null);
+    }
+}
diff --git a/render-test/android/app/src/main/AndroidManifest.xml b/render-test/android/app/src/main/AndroidManifest.xml
index 6c7af7ed8f..8df48ef97f 100644
--- a/render-test/android/app/src/main/AndroidManifest.xml
+++ b/render-test/android/app/src/main/AndroidManifest.xml
@@ -9,7 +9,7 @@
       android:fullBackupContent="false"
       android:icon="@mipmap/ic_launcher"
       android:label="@string/app_name"
-      android:hasCode="false">
+      android:hasCode="true">
 
     <activity android:name="android.app.NativeActivity"
               android:label="@string/app_name"
diff --git a/render-test/android/app/src/main/assets/res.zip b/render-test/android/app/src/main/assets/res.zip
deleted file mode 100644
index 533c4f7fc1..0000000000
--- a/render-test/android/app/src/main/assets/res.zip
+++ /dev/null
diff --git a/render-test/android/app/src/main/assets/to_zip.txt b/render-test/android/app/src/main/assets/to_zip.txt
new file mode 100644
index 0000000000..7023ad9cfa
--- /dev/null
+++ b/render-test/android/app/src/main/assets/to_zip.txt
@@ -0,0 +1,15 @@
+mapbox-gl-js/test/integration/data/
+mapbox-gl-js/test/integration/video/
+mapbox-gl-js/test/integration/tilesets/
+mapbox-gl-js/test/integration/tiles/
+mapbox-gl-js/test/integration/styles/
+mapbox-gl-js/test/integration/render-tests/
+mapbox-gl-js/test/integration/query-tests/
+mapbox-gl-js/test/integration/image/
+mapbox-gl-js/test/integration/glyphs/
+mapbox-gl-js/test/integration/geojson/
+mapbox-gl-js/test/integration/sprites/
+vendor/mapbox-gl-styles/styles/
+vendor/mapbox-gl-styles/sprites/
+ignores/
+android-manifest.json
\ No newline at end of file
diff --git a/render-test/android/render_test_setup.sh b/render-test/android/render_test_setup.sh
deleted file mode 100755
index 1dea44399e..0000000000
--- a/render-test/android/render_test_setup.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-adb shell rm -rf /sdcard/render-test
-adb shell mkdir /sdcard/render-test
-adb shell mkdir /sdcard/render-test/vendor
-adb shell mkdir /sdcard/render-test/ignores
-adb shell mkdir /sdcard/render-test/render-test/tests
-
-# push test sources
-adb push ../../mapbox-gl-js/test/integration/render-tests /sdcard/render-test/mapbox-gl-js/test/integration/render-tests
-adb push ../../mapbox-gl-js/test/integration/query-tests /sdcard/render-test/mapbox-gl-js/test/integration/query-tests 
-adb push ../../mapbox-gl-js/test/integration/tiles /sdcard/render-test/mapbox-gl-js/test/integration/tiles
-adb push ../../mapbox-gl-js/test/integration/glyphs /sdcard/render-test/mapbox-gl-js/test/integration/glyphs
-adb push ../../mapbox-gl-js/test/integration/styles /sdcard/render-test/mapbox-gl-js/test/integration/styles 
-adb push ../../mapbox-gl-js/test/integration/tilesets /sdcard/render-test/mapbox-gl-js/test/integration/tilesets 
-adb push ../../mapbox-gl-js/test/integration/image /sdcard/render-test/mapbox-gl-js/test/integration/image 
-adb push ../../mapbox-gl-js/test/integration/video /sdcard/render-test/mapbox-gl-js/test/integration/video 
-adb push ../../vendor/mapbox-gl-styles/styles /sdcard/render-test/vendor/mapbox-gl-styles/styles 
-adb push ../../vendor/mapbox-gl-styles/sprites /sdcard/render-test/vendor/mapbox-gl-styles/sprites 
-adb push ../../mapbox-gl-js/test/integration/data /sdcard/render-test/mapbox-gl-js/test/integration/data 
-adb push ../../mapbox-gl-js/test/integration/geojson /sdcard/render-test/mapbox-gl-js/test/integration/geojson
-mkdir sprites
-cp -r ../../mapbox-gl-js/test/integration/sprites/ sprites
-adb push sprites /sdcard/render-test/mapbox-gl-js/test/integration/sprites
-rm -rf sprites
-
-# push extra expectations
-adb push ../../render-test/expected/render-tests /sdcard/render-test/render-test/expected/render-tests
-
-# push default ignore lists
-adb shell mkdir /sdcard/render-test/platform
-adb shell mkdir /sdcard/render-test/platform/node
-adb shell mkdir /sdcard/render-test/platform/node/test
-adb push ../../platform/node/test/ignores.json /sdcard/render-test/platform/node/test
-adb shell mkdir /sdcard/render-test/render-test
-adb push ../linux-ignores.json /sdcard/render-test/render-test
-adb push ../tests/should-fail.json /sdcard/render-test/render-test/tests
-
-# push manifest
-adb push ../android-manifest.json /sdcard/render-test
-
-adb shell ls /sdcard/render-test/
\ No newline at end of file
diff --git a/render-test/manifest_parser.cpp b/render-test/manifest_parser.cpp
index f1884634e5..8b5a534027 100644
--- a/render-test/manifest_parser.cpp
+++ b/render-test/manifest_parser.cpp
@@ -34,9 +34,15 @@ const std::vector<std::pair<std::string, std::string>>& Manifest::getIgnores() c
 const std::string& Manifest::getTestRootPath() const {
     return testRootPath;
 }
+const std::string& Manifest::getAssetPath() const {
+    return assetPath;
+}
 const std::string& Manifest::getManifestPath() const {
     return manifestPath;
 }
+const std::string& Manifest::getResultPath() const {
+    return resultPath;
+}
 
 void Manifest::doShuffle(uint32_t seed) {
     std::seed_seq sequence{seed};
@@ -239,7 +245,7 @@ mbgl::filesystem::path getValidPath(const std::string& manifestPath, const std::
         result = BasePath / result;
     }
     if (mbgl::filesystem::exists(result)) {
-        return result;
+        return result.lexically_normal();
     }
     mbgl::Log::Warning(mbgl::Event::General, "Invalid path is provoided inside the manifest file: %s", path.c_str());
     return mbgl::filesystem::path{};
@@ -285,6 +291,18 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manife
             return mbgl::nullopt;
         }
     }
+    if (document.HasMember("result_path")) {
+        const auto& resultPathValue = document["result_path"];
+        if (!resultPathValue.IsString()) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Invalid assetPath is provoided inside the manifest file: %s", filePath.c_str());
+            return mbgl::nullopt;
+        }
+        manifest.resultPath = (getValidPath(manifest.manifestPath, resultPathValue.GetString()) / "").string();
+        if (manifest.resultPath.empty()) {
+            return mbgl::nullopt;
+        }
+    }
     mbgl::filesystem::path baseTestPath;
     if (document.HasMember("base_test_path")) {
         const auto& testPathValue = document["base_test_path"];
@@ -366,6 +384,11 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manife
     if (manifest.manifestPath.back() == '/') {
         manifest.manifestPath.pop_back();
     }
+    if (manifest.resultPath.empty()) {
+        manifest.resultPath = manifest.manifestPath;
+    } else if (manifest.resultPath.back() == '/') {
+        manifest.resultPath.pop_back();
+    }
 
     std::vector<mbgl::filesystem::path> paths;
     for (const auto& id : testNames) {
diff --git a/render-test/manifest_parser.hpp b/render-test/manifest_parser.hpp
index bc5adf1091..c4672fb4c5 100644
--- a/render-test/manifest_parser.hpp
+++ b/render-test/manifest_parser.hpp
@@ -17,7 +17,9 @@ public:
     const std::vector<std::pair<std::string, std::string>>& getIgnores() const;
     const std::vector<TestPaths>& getTestPaths() const;
     const std::string& getTestRootPath() const;
+    const std::string& getAssetPath() const;
     const std::string& getManifestPath() const;
+    const std::string& getResultPath() const;
     void doShuffle(uint32_t seed);
 
     std::string localizeURL(const std::string& url) const;
@@ -43,6 +45,7 @@ private:
     std::string testRootPath;
     std::string vendorPath;
     std::string assetPath;
+    std::string resultPath;
     std::vector<std::pair<std::string, std::string>> ignores;
     std::vector<TestPaths> testPaths;
 };
diff --git a/render-test/render_test.cpp b/render-test/render_test.cpp
index 5ea1202526..e89cab1c1b 100644
--- a/render-test/render_test.cpp
+++ b/render-test/render_test.cpp
@@ -182,29 +182,25 @@ int runRenderTests(int argc, char** argv) {
                 status = "passed";
                 color = "green";
                 stats.passedTests++;
-                mbgl::Log::Warning(mbgl::Event::General, "* passed %s" ANSI_COLOR_RESET "\n", id.c_str());
                 printf(ANSI_COLOR_GREEN "* passed %s" ANSI_COLOR_RESET "\n", id.c_str());
             } else if (errored) {
                 status = "errored";
                 color = "red";
                 stats.erroredTests++;
-                mbgl::Log::Warning(mbgl::Event::General, "* errored %s" ANSI_COLOR_RESET "\n", id.c_str());
                 printf(ANSI_COLOR_RED "* errored %s" ANSI_COLOR_RESET "\n", id.c_str());
                 printf(ANSI_COLOR_RED "* error: %s" ANSI_COLOR_RESET "\n", metadata.errorMessage.c_str());
             } else {
                 status = "failed";
                 color = "red";
                 stats.failedTests++;
-                mbgl::Log::Warning(mbgl::Event::General, "* failed %s" ANSI_COLOR_RESET "\n", id.c_str());
                 printf(ANSI_COLOR_RED "* failed %s" ANSI_COLOR_RESET "\n", id.c_str());
             }
         }
 
         metadatas.push_back(std::move(metadata));
     }
-const auto& testRootPath = manifest.getManifestPath();
     const auto resultPath =
-        testRootPath + "/" + (testNames.empty() ? "render-tests" : testNames.front()) + "_index.html";
+        manifest.getResultPath() + "/" + (testNames.empty() ? "render-tests" : testNames.front()) + "_index.html";
     std::string resultsHTML = createResultPage(stats, metadatas, shuffle, seed);
     mbgl::util::write_file(resultPath, resultsHTML);
 
@@ -212,23 +208,18 @@ const auto& testRootPath = manifest.getManifestPath();
         stats.erroredTests + stats.failedTests + stats.ignoreFailedTests + stats.ignorePassedTests + stats.passedTests;
 
     if (stats.passedTests) {
-        mbgl::Log::Warning(mbgl::Event::General, "%u passed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.passedTests, 100.0 * stats.passedTests / count);
         printf(ANSI_COLOR_GREEN "%u passed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.passedTests, 100.0 * stats.passedTests / count);
     }
     if (stats.ignorePassedTests) {
-        mbgl::Log::Warning(mbgl::Event::General,"%u passed but were ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.ignorePassedTests, 100.0 * stats.ignorePassedTests / count);
         printf(ANSI_COLOR_YELLOW "%u passed but were ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.ignorePassedTests, 100.0 * stats.ignorePassedTests / count);
     }
     if (stats.ignoreFailedTests) {
-        mbgl::Log::Warning(mbgl::Event::General,"%u ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.ignoreFailedTests, 100.0 * stats.ignoreFailedTests / count);
         printf(ANSI_COLOR_LIGHT_GRAY "%u ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.ignoreFailedTests, 100.0 * stats.ignoreFailedTests / count);
     }
     if (stats.failedTests) {
-        mbgl::Log::Warning(mbgl::Event::General, "%u failed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.failedTests, 100.0 * stats.failedTests / count);
         printf(ANSI_COLOR_RED "%u failed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.failedTests, 100.0 * stats.failedTests / count);
     }
     if (stats.erroredTests) {
-        mbgl::Log::Warning(mbgl::Event::General,"%u errored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.erroredTests, 100.0 * stats.erroredTests / count);
         printf(ANSI_COLOR_RED "%u errored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.erroredTests, 100.0 * stats.erroredTests / count);
     }
 
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 8a4b0b3b0e..9d4f378621 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -596,7 +596,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata, R
         std::string imagePath = operationArray[2].GetString();
         imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end());
 
-        const mbgl::filesystem::path filePath = mbgl::filesystem::path(manifest.getTestRootPath()) / imagePath;
+        const mbgl::filesystem::path filePath = (mbgl::filesystem::path(manifest.getAssetPath()) / imagePath);
 
         mbgl::optional<std::string> maybeImage = mbgl::util::readFile(filePath.string());
         if (!maybeImage) {