[render-test] Add manifest parser for render-test-runner

14 files changed, 326 insertions, 195 deletions

diff --git a/next/platform/linux/linux.cmake b/next/platform/linux/linux.cmake
index 5bde30a61a..6ea1d4eb9f 100644
--- a/next/platform/linux/linux.cmake
+++ b/next/platform/linux/linux.cmake
@@ -144,14 +144,9 @@ add_test(
         render-tests
         --recycle-map
         --shuffle
-        --expectationsPath=render-test/expected/render-tests
+        --rootPath=${MBGL_ROOT}/render-test
         --seed=${MBGL_RENDER_TEST_SEED}
-    WORKING_DIRECTORY ${MBGL_ROOT}
 )
 
-add_test(
-    NAME mbgl-render-test-probes
-    COMMAND mbgl-render-test-runner tests --rootPath=${MBGL_ROOT}/render-test
-    WORKING_DIRECTORY ${MBGL_ROOT}
-)
-add_test(NAME mbgl-query-test COMMAND mbgl-render-test-runner query-tests WORKING_DIRECTORY ${MBGL_ROOT})
+add_test(NAME mbgl-render-test-probes COMMAND mbgl-render-test-runner tests --rootPath=${MBGL_ROOT}/render-test)
+add_test(NAME mbgl-query-test COMMAND mbgl-render-test-runner query-tests --rootPath=${MBGL_ROOT}/render-test)
diff --git a/next/platform/macos/macos.cmake b/next/platform/macos/macos.cmake
index cfabcb1cfa..4838459c18 100644
--- a/next/platform/macos/macos.cmake
+++ b/next/platform/macos/macos.cmake
@@ -208,19 +208,10 @@ add_test(
         render-tests
         --recycle-map
         --shuffle
-        --expectationsPath=render-test/expected/render-tests
+        --rootPath=${MBGL_ROOT}/render-test
         --seed=${MBGL_RENDER_TEST_SEED}
-    WORKING_DIRECTORY ${MBGL_ROOT}
 )
 
-add_test(
-    NAME mbgl-render-test-probes
-    COMMAND
-        mbgl-render-test-runner
-        tests
-        --rootPath=${MBGL_ROOT}/render-test
-        --expectationsPath=tests/mac
-    WORKING_DIRECTORY ${MBGL_ROOT}
-)
+add_test(NAME mbgl-render-test-probes COMMAND mbgl-render-test-runner tests --rootPath=${MBGL_ROOT}/render-test)
 
-add_test(NAME mbgl-query-test COMMAND mbgl-render-test-runner query-tests WORKING_DIRECTORY ${MBGL_ROOT})
+add_test(NAME mbgl-query-test COMMAND mbgl-render-test-runner query-tests --rootPath=${MBGL_ROOT}/render-test)
diff --git a/next/render-test/CMakeLists.txt b/next/render-test/CMakeLists.txt
index 02ee6141cf..fa4bb0d615 100644
--- a/next/render-test/CMakeLists.txt
+++ b/next/render-test/CMakeLists.txt
@@ -8,6 +8,8 @@ add_library(
     ${MBGL_ROOT}/render-test/file_source.hpp
     ${MBGL_ROOT}/render-test/filesystem.hpp
     ${MBGL_ROOT}/render-test/include/mbgl/render_test.hpp
+    ${MBGL_ROOT}/render-test/manifest_parser.cpp
+    ${MBGL_ROOT}/render-test/manifest_parser.hpp
     ${MBGL_ROOT}/render-test/metadata.hpp
     ${MBGL_ROOT}/render-test/parser.cpp
     ${MBGL_ROOT}/render-test/parser.hpp
diff --git a/render-test/android-manifest.json b/render-test/android-manifest.json
new file mode 100644
index 0000000000..5144c747d1
--- /dev/null
+++ b/render-test/android-manifest.json
@@ -0,0 +1,8 @@
+{
+    "base_test_path":"mapbox-gl-js/test/integration",
+    "probe_test_path":"",
+    "expectation_paths":["render-test/expected/render-tests"],
+    "ignore_paths":["platform/node/test/ignores.json", "render-test/linux-ignores.json", "render-test/tests/should-fail.json"],
+    "vendor_path":"vendor",
+    "asset_path": "mapbox-gl-js/test/integration"
+}
\ No newline at end of file
diff --git a/render-test/android/render_test_setup.sh b/render-test/android/render_test_setup.sh
index e34d71df3b..1dea44399e 100755
--- a/render-test/android/render_test_setup.sh
+++ b/render-test/android/render_test_setup.sh
@@ -3,8 +3,8 @@
 adb shell rm -rf /sdcard/render-test
 adb shell mkdir /sdcard/render-test
 adb shell mkdir /sdcard/render-test/vendor
-adb shell mkdir /sdcard/render-test/expected
 adb shell mkdir /sdcard/render-test/ignores
+adb shell mkdir /sdcard/render-test/render-test/tests
 
 # push test sources
 adb push ../../mapbox-gl-js/test/integration/render-tests /sdcard/render-test/mapbox-gl-js/test/integration/render-tests
@@ -25,7 +25,7 @@ adb push sprites /sdcard/render-test/mapbox-gl-js/test/integration/sprites
 rm -rf sprites
 
 # push extra expectations
-adb push ../../render-test/expected /sdcard/render-test/render-test/expected
+adb push ../../render-test/expected/render-tests /sdcard/render-test/render-test/expected/render-tests
 
 # push default ignore lists
 adb shell mkdir /sdcard/render-test/platform
@@ -33,6 +33,10 @@ adb shell mkdir /sdcard/render-test/platform/node
 adb shell mkdir /sdcard/render-test/platform/node/test
 adb push ../../platform/node/test/ignores.json /sdcard/render-test/platform/node/test
 adb shell mkdir /sdcard/render-test/render-test
-adb push ../../render-test/linux-ignores.json /sdcard/render-test/render-test
+adb push ../linux-ignores.json /sdcard/render-test/render-test
+adb push ../tests/should-fail.json /sdcard/render-test/render-test/tests
+
+# push manifest
+adb push ../android-manifest.json /sdcard/render-test
 
 adb shell ls /sdcard/render-test/
\ No newline at end of file
diff --git a/render-test/linux-manifest.json b/render-test/linux-manifest.json
new file mode 100644
index 0000000000..4cb2a699b4
--- /dev/null
+++ b/render-test/linux-manifest.json
@@ -0,0 +1,8 @@
+{
+    "base_test_path":"../mapbox-gl-js/test/integration",
+    "probe_test_path":"",
+    "expectation_paths":["expected/render-tests"],
+    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json", "../render-test/tests/should-fail.json"],
+    "vendor_path":"../vendor",
+    "asset_path": "../mapbox-gl-js/test/integration"
+}
\ No newline at end of file
diff --git a/render-test/mac-manifest.json b/render-test/mac-manifest.json
new file mode 100644
index 0000000000..c54cc958af
--- /dev/null
+++ b/render-test/mac-manifest.json
@@ -0,0 +1,8 @@
+{
+    "base_test_path":"../mapbox-gl-js/test/integration",
+    "probe_test_path":"",
+    "expectation_paths":["expected/render-tests", "tests/mac"],
+    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/mac-ignores.json", "../render-test/tests/should-fail.json"],
+    "vendor_path":"../vendor",
+    "asset_path": "../mapbox-gl-js/test/integration"
+}
\ No newline at end of file
diff --git a/render-test/manifest_parser.cpp b/render-test/manifest_parser.cpp
new file mode 100644
index 0000000000..458df137c8
--- /dev/null
+++ b/render-test/manifest_parser.cpp
@@ -0,0 +1,158 @@
+#include "manifest_parser.hpp"
+#include <mbgl/util/logging.hpp>
+#include <mbgl/util/rapidjson.hpp>
+#include "parser.hpp"
+
+mbgl::filesystem::path ManifestParser::getValidPath(const std::string& path) {
+    const static mbgl::filesystem::path BasePath{rootPath};
+    mbgl::filesystem::path result{path};
+    if (result.is_relative()) {
+        result = BasePath / result;
+    }
+    if (mbgl::filesystem::exists(result)) {
+        return result;
+    }
+    mbgl::Log::Warning(mbgl::Event::General, "Invalid path is provoided inside the manifest file: %s", path.c_str());
+    return mbgl::filesystem::path{};
+}
+
+const std::string ManifestParser::getVendorPath() const {
+    return vendorPath;
+}
+const std::string ManifestParser::getIntegrationPath() const {
+    return assetPath;
+}
+const mbgl::filesystem::path ManifestParser::getBaseTestPath() const {
+    return baseTestPath;
+}
+const mbgl::filesystem::path ManifestParser::getProbeTestPath() const {
+    return probeTestPath;
+}
+const std::vector<mbgl::filesystem::path> ManifestParser::getExpectationsPaths() const {
+    return expectationPaths;
+}
+const std::vector<mbgl::filesystem::path> ManifestParser::getIgnoresPaths() const {
+    return ignorePaths;
+}
+const std::string ManifestParser::getTestPath() const {
+    assert(!testPath.empty());
+    return testPath;
+}
+void ManifestParser::setTestPath(const std::string& testPath_) {
+    testPath = testPath_;
+    if (testPath.back() == '/') {
+        testPath.pop_back();
+    }
+}
+
+bool ManifestParser::parseManifest(const std::string& rootPath_) {
+    rootPath = rootPath_;
+    auto filePath = rootPath;
+#ifdef __APPLE__
+    filePath += "/mac-manifest.json";
+#elif __ANDROID__
+    filePath += "/android-manifest.json";
+#else
+    filePath += "/linux-manifest.json";
+#endif
+    if (!mbgl::filesystem::exists(filePath)) {
+        mbgl::Log::Error(mbgl::Event::General, "Provided manifest file: %s is not valid", filePath.c_str());
+        return false;
+    }
+    auto contents = readJson(filePath);
+    if (!contents.is<mbgl::JSDocument>()) {
+        mbgl::Log::Error(mbgl::Event::General, "Provided manifest file: %s is not a valid json", filePath.c_str());
+        return false;
+    }
+    auto document = std::move(contents.get<mbgl::JSDocument>());
+    if (document.HasMember("asset_path")) {
+        const auto& assetPathValue = document["asset_path"];
+        if (!assetPathValue.IsString()) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Invalid assetPath is provoided inside the manifest file: %s", filePath.c_str());
+            return false;
+        }
+
+        assetPath = getValidPath(assetPathValue.GetString()).string();
+        if (assetPath.back() != '/') {
+            assetPath.push_back('/');
+        }
+    }
+    if (document.HasMember("vendor_path")) {
+        const auto& vendorPathValue = document["vendor_path"];
+        if (!vendorPathValue.IsString()) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Invalid vendorPath is provoided inside the manifest file: %s", filePath.c_str());
+            return false;
+        }
+        auto path = std::string(vendorPathValue.GetString());
+
+        vendorPath = getValidPath(vendorPathValue.GetString()).string();
+        if (vendorPath.back() != '/') {
+            vendorPath.push_back('/');
+        }
+    }
+    if (document.HasMember("base_test_path")) {
+        const auto& testPathValue = document["base_test_path"];
+        if (!testPathValue.IsString()) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Invalid testPath is provoided inside the manifest file: %s", filePath.c_str());
+            return false;
+        }
+        auto path = getValidPath(testPathValue.GetString()).string();
+        if (path.back() == '/') {
+            path.pop_back();
+        }
+        baseTestPath = mbgl::filesystem::path(path);
+        setTestPath(baseTestPath.string());
+    }
+    if (document.HasMember("probe_test_path")) {
+        const auto& testPathValue = document["probe_test_path"];
+        if (!testPathValue.IsString()) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Invalid testPath is provoided inside the manifest file: %s", filePath.c_str());
+            return false;
+        }
+        auto path = getValidPath(testPathValue.GetString()).string();
+        if (path.back() == '/') {
+            path.pop_back();
+        }
+        probeTestPath = mbgl::filesystem::path(path);
+    }
+    if (document.HasMember("expectation_paths")) {
+        const auto& expectationPathValue = document["expectation_paths"];
+        if (!expectationPathValue.IsArray()) {
+            mbgl::Log::Warning(mbgl::Event::General,
+                               "Invalid expectationPath is provoided inside the manifest file: %s",
+                               filePath.c_str());
+            return false;
+        }
+        for (const auto& value : expectationPathValue.GetArray()) {
+            if (!value.IsString()) {
+                return false;
+            }
+            expectationPaths.emplace_back(getValidPath(value.GetString()));
+            if (expectationPaths.back().empty()) {
+                return false;
+            }
+        }
+    }
+    if (document.HasMember("ignore_paths")) {
+        const auto& ignorePathValue = document["ignore_paths"];
+        if (!ignorePathValue.IsArray()) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Invalid ignorePath is provoided inside the manifest file: %s", filePath.c_str());
+            return false;
+        }
+        for (const auto& value : ignorePathValue.GetArray()) {
+            if (!value.IsString()) {
+                return false;
+            }
+            ignorePaths.emplace_back(getValidPath(value.GetString()));
+            if (ignorePaths.back().empty()) {
+                return false;
+            }
+        }
+    }
+    return true;
+}
diff --git a/render-test/manifest_parser.hpp b/render-test/manifest_parser.hpp
new file mode 100644
index 0000000000..fd7b81d9c8
--- /dev/null
+++ b/render-test/manifest_parser.hpp
@@ -0,0 +1,35 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include "filesystem.hpp"
+
+class ManifestParser {
+public:
+    static ManifestParser& getInstance() {
+        static ManifestParser instance;
+        return instance;
+    }
+    const std::string getVendorPath() const;
+    const std::string getIntegrationPath() const;
+    const mbgl::filesystem::path getBaseTestPath() const;
+    const mbgl::filesystem::path getProbeTestPath() const;
+    const std::vector<mbgl::filesystem::path> getExpectationsPaths() const;
+    const std::vector<mbgl::filesystem::path> getIgnoresPaths() const;
+    bool parseManifest(const std::string& rootPath_);
+    const std::string getTestPath() const;
+    void setTestPath(const std::string& testPath_);
+
+private:
+    ManifestParser() = default;
+    mbgl::filesystem::path getValidPath(const std::string& path);
+
+    std::string rootPath{};
+    std::string testPath{};
+    std::string vendorPath{};
+    std::string assetPath{};
+    mbgl::filesystem::path baseTestPath{};
+    mbgl::filesystem::path probeTestPath{};
+    std::vector<mbgl::filesystem::path> expectationPaths{};
+    std::vector<mbgl::filesystem::path> ignorePaths{};
+};
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index 11a42f3202..c56fc7ca19 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -19,12 +19,13 @@
 #include <boost/archive/iterators/ostream_iterator.hpp>
 
 #include "filesystem.hpp"
+#include "manifest_parser.hpp"
 #include "metadata.hpp"
 #include "parser.hpp"
 #include "runner.hpp"
 
-#include <sstream>
 #include <regex>
+#include <sstream>
 
 namespace {
 
@@ -96,13 +97,8 @@ std::string prependFileScheme(const std::string &url) {
     return fileScheme + url;
 }
 
-mbgl::optional<std::string> getVendorPath(const std::string& url,
-                                          const std::regex& regex,
-                                          const std::string& testRootPath,
-                                          bool glyphsPath = false) {
-    static const mbgl::filesystem::path vendorPath = getValidPath(testRootPath, std::string("vendor/"));
-
-    mbgl::filesystem::path file = std::regex_replace(url, regex, vendorPath.string());
+mbgl::optional<std::string> getVendorPath(const std::string& url, const std::regex& regex, bool glyphsPath = false) {
+    mbgl::filesystem::path file = std::regex_replace(url, regex, ManifestParser::getInstance().getVendorPath());
     if (mbgl::filesystem::exists(file.parent_path())) {
         return removeURLArguments(file.string());
     }
@@ -117,12 +113,9 @@ mbgl::optional<std::string> getVendorPath(const std::string& url,
 mbgl::optional<std::string> getIntegrationPath(const std::string& url,
                                                const std::string& parent,
                                                const std::regex& regex,
-                                               const std::string& testRootPath,
                                                bool glyphsPath = false) {
-    static const mbgl::filesystem::path integrationPath =
-        getValidPath(testRootPath, std::string("mapbox-gl-js/test/integration/"));
-
-    mbgl::filesystem::path file = std::regex_replace(url, regex, integrationPath.string() + parent);
+    mbgl::filesystem::path file =
+        std::regex_replace(url, regex, ManifestParser::getInstance().getIntegrationPath() + parent);
     if (mbgl::filesystem::exists(file.parent_path())) {
         return removeURLArguments(file.string());
     }
@@ -134,48 +127,46 @@ mbgl::optional<std::string> getIntegrationPath(const std::string& url,
     return {};
 }
 
-mbgl::optional<std::string> localizeLocalURL(const std::string& url,
-                                             const std::string& testRootPath,
-                                             bool glyphsPath = false) {
+mbgl::optional<std::string> localizeLocalURL(const std::string& url, bool glyphsPath = false) {
     static const std::regex regex{"local://"};
-    if (auto vendorPath = getVendorPath(url, regex, testRootPath, glyphsPath)) {
+    if (auto vendorPath = getVendorPath(url, regex, glyphsPath)) {
         return vendorPath;
     } else {
-        return getIntegrationPath(url, "", regex, testRootPath, glyphsPath);
+        return getIntegrationPath(url, "", regex, glyphsPath);
     }
 }
 
-mbgl::optional<std::string> localizeHttpURL(const std::string& url, const std::string& testRootPath) {
+mbgl::optional<std::string> localizeHttpURL(const std::string& url) {
     static const std::regex regex{"http://localhost:2900"};
-    if (auto vendorPath = getVendorPath(url, regex, testRootPath)) {
+    if (auto vendorPath = getVendorPath(url, regex)) {
         return vendorPath;
     } else {
-        return getIntegrationPath(url, "", regex, testRootPath);
+        return getIntegrationPath(url, "", regex);
     }
 }
 
-mbgl::optional<std::string> localizeMapboxSpriteURL(const std::string& url, const std::string& testRootPath) {
+mbgl::optional<std::string> localizeMapboxSpriteURL(const std::string& url) {
     static const std::regex regex{"mapbox://"};
-    return getIntegrationPath(url, "", regex, testRootPath);
+    return getIntegrationPath(url, "", regex);
 }
 
-mbgl::optional<std::string> localizeMapboxFontsURL(const std::string& url, const std::string& testRootPath) {
+mbgl::optional<std::string> localizeMapboxFontsURL(const std::string& url) {
     static const std::regex regex{"mapbox://fonts"};
-    return getIntegrationPath(url, "glyphs/", regex, testRootPath, true);
+    return getIntegrationPath(url, "glyphs/", regex, true);
 }
 
-mbgl::optional<std::string> localizeMapboxTilesURL(const std::string& url, const std::string& testRootPath) {
+mbgl::optional<std::string> localizeMapboxTilesURL(const std::string& url) {
     static const std::regex regex{"mapbox://"};
-    if (auto vendorPath = getVendorPath(url, regex, testRootPath)) {
+    if (auto vendorPath = getVendorPath(url, regex)) {
         return vendorPath;
     } else {
-        return getIntegrationPath(url, "tiles/", regex, testRootPath);
+        return getIntegrationPath(url, "tiles/", regex);
     }
 }
 
-mbgl::optional<std::string> localizeMapboxTilesetURL(const std::string& url, const std::string& testRootPath) {
+mbgl::optional<std::string> localizeMapboxTilesetURL(const std::string& url) {
     static const std::regex regex{"mapbox://"};
-    return getIntegrationPath(url, "tilesets/", regex, testRootPath);
+    return getIntegrationPath(url, "tilesets/", regex);
 }
 
 void writeJSON(rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer, const mbgl::Value& value) {
@@ -205,33 +196,6 @@ void writeJSON(rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer, const m
 
 } // namespace
 
-static const mbgl::filesystem::path DefaultRootPath{std::string(TEST_RUNNER_ROOT_PATH)};
-
-const mbgl::filesystem::path getValidPath(const std::string& basePath, const std::string& subPath) {
-    auto filePath = mbgl::filesystem::path(basePath) / subPath;
-    if (mbgl::filesystem::exists(filePath)) {
-        return filePath;
-    }
-    // Fall back to check default path
-    filePath = DefaultRootPath / subPath;
-    if (mbgl::filesystem::exists(filePath)) {
-        return filePath;
-    }
-    mbgl::Log::Warning(mbgl::Event::General, "Failed to find path: %s", subPath.c_str());
-    return mbgl::filesystem::path{};
-}
-
-/// Returns path of the render test cases directory.
-const std::string getTestPath(const std::string& rootTestPath) {
-    // Check if sub-directory exits or not
-    auto testBasePath = mbgl::filesystem::path(rootTestPath) / ("mapbox-gl-js/test/integration");
-    if (mbgl::filesystem::exists(testBasePath)) {
-        return testBasePath.string();
-    }
-    // Use root test path for further processing
-    return rootTestPath;
-}
-
 std::string toJSON(const mbgl::Value& value, unsigned indent, bool singleLine) {
     rapidjson::StringBuffer buffer;
     rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
@@ -418,14 +382,9 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
 
     args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", {'r', "recycle-map"});
     args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order", {'s', "shuffle"});
-    args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)",
-                                        { "seed" });
+    args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)", {"seed"});
     args::ValueFlag<std::string> testPathValue(argumentParser, "rootPath", "Test root rootPath", {'p', "rootPath"});
     args::ValueFlag<std::regex> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"});
-    args::ValueFlag<std::string> expectationsPathValue(
-        argumentParser, "expectationsPath", "Test expectations path", {'e', "expectationsPath"});
-    args::ValueFlag<std::string> ignoresPathValue(
-        argumentParser, "ignoresPath", "Test ignore list path", {'i', "ignoresPath"});
     args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
 
     try {
@@ -459,39 +418,31 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
             mbgl::Event::General, "Provided test rootPath '%s' does not exist.", rootPath.string().c_str());
         exit(4);
     }
-    std::vector<mbgl::filesystem::path> expectationsPaths;
-    if (expectationsPathValue) {
-        auto expectationsPath = mbgl::filesystem::path(testRootPath) / args::get(expectationsPathValue);
-        if (!mbgl::filesystem::exists(expectationsPath)) {
-            mbgl::Log::Error(mbgl::Event::General,
-                             "Provided expectationsPath '%s' does not exist.",
-                             expectationsPath.string().c_str());
-            exit(5);
-        }
-        expectationsPaths.emplace_back(std::move(expectationsPath));
-    }
 
-    std::string ignoresPath{};
-    if (ignoresPathValue) {
-        auto path = mbgl::filesystem::path(testRootPath) / args::get(ignoresPathValue);
-        if (!mbgl::filesystem::exists(path)) {
-            mbgl::Log::Error(
-                mbgl::Event::General, "Provided ignore list path '%s' does not exist.", path.string().c_str());
-            exit(6);
-        }
-        ignoresPath = path.string();
+    if (!ManifestParser::getInstance().parseManifest(testRootPath)) {
+        mbgl::Log::Error(
+            mbgl::Event::General, "Provided test rootPath '%s' does not exist.", rootPath.string().c_str());
+        exit(5);
     }
 
     std::vector<mbgl::filesystem::path> paths;
-    auto testBasePath = mbgl::filesystem::path(getTestPath(testRootPath));
-    for (const auto& id : args::get(testNameValues)) {
-        paths.emplace_back(testBasePath / id);
-    }
-
-    if (paths.empty()) {
-        paths.emplace_back(testBasePath);
+    const auto& baseTestPath = ManifestParser::getInstance().getBaseTestPath();
+    const auto& probeTestPath = ManifestParser::getInstance().getProbeTestPath();
+    if (!baseTestPath.empty()) {
+        for (const auto& id : args::get(testNameValues)) {
+            if (id == "tests") {
+                ManifestParser::getInstance().setTestPath(probeTestPath);
+                paths.emplace_back(probeTestPath / id);
+            } else {
+                paths.emplace_back(baseTestPath / id);
+            }
+        }
+        if (paths.empty()) {
+            paths.emplace_back(baseTestPath);
+        }
     }
 
+    const std::vector<mbgl::filesystem::path>& expectationsPaths = ManifestParser::getInstance().getExpectationsPaths();
     // Recursively traverse through the test paths and collect test directories containing "style.json".
     std::vector<TestPaths> testPaths;
     testPaths.reserve(paths.size());
@@ -515,40 +466,21 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
     return ArgumentsTuple{recycleMapFlag ? args::get(recycleMapFlag) : false,
                           shuffleFlag ? args::get(shuffleFlag) : false,
                           seedValue ? args::get(seedValue) : 1u,
-                          testRootPath,
-                          ignoresPath,
                           std::move(testPaths)};
 }
 
-std::vector<std::pair<std::string, std::string>> parseIgnores(const std::string& testRootPath,
-                                                              const std::string& ignoresPath) {
+std::vector<std::pair<std::string, std::string>> parseIgnores() {
     std::vector<std::pair<std::string, std::string>> ignores;
-    auto mainIgnoresPath = getValidPath(testRootPath, "platform/node/test/ignores.json");
-
-    mbgl::filesystem::path platformSpecificIgnores;
-    mbgl::filesystem::path ownTestsIgnores = getValidPath(testRootPath, "render-test/tests/should-fail.json");
-
-#ifdef __APPLE__
-    platformSpecificIgnores = getValidPath(testRootPath, "render-test/mac-ignores.json");
-#elif __linux__
-    platformSpecificIgnores = getValidPath(testRootPath, "render-test/linux-ignores.json");
-#endif
-
-    std::vector<mbgl::filesystem::path> ignoresPaths = {mainIgnoresPath, platformSpecificIgnores, ownTestsIgnores};
+    const std::vector<mbgl::filesystem::path>& ignoresPaths = ManifestParser::getInstance().getIgnoresPaths();
 
-    if (!ignoresPath.empty()) {
-        ignoresPaths.emplace_back(ignoresPath);
-    }
     for (const auto& path : ignoresPaths) {
         auto maybeIgnores = readJson(path);
         if (!maybeIgnores.is<mbgl::JSDocument>()) {
             continue;
         }
         for (const auto& property : maybeIgnores.get<mbgl::JSDocument>().GetObject()) {
-            const std::string ignore = { property.name.GetString(),
-                                         property.name.GetStringLength() };
-            const std::string reason = { property.value.GetString(),
-                                         property.value.GetStringLength() };
+            const std::string ignore = {property.name.GetString(), property.name.GetStringLength()};
+            const std::string reason = {property.value.GetString(), property.value.GetStringLength()};
             ignores.emplace_back(std::make_pair(ignore, reason));
         }
     }
@@ -601,7 +533,7 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
             std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()};
             assert(!mark.empty());
             result.memory.emplace(std::piecewise_construct,
-                                  std::forward_as_tuple(std::move(mark)), 
+                                  std::forward_as_tuple(std::move(mark)),
                                   std::forward_as_tuple(probeValue[1].GetUint64(), probeValue[2].GetUint64()));
         }
     }
@@ -645,7 +577,7 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
     return result;
 }
 
-TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRootPath) {
+TestMetadata parseTestMetadata(const TestPaths& paths) {
     TestMetadata metadata;
     metadata.paths = paths;
 
@@ -656,7 +588,7 @@ TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRo
     }
 
     metadata.document = std::move(maybeJson.get<mbgl::JSDocument>());
-    localizeStyleURLs(metadata.document, metadata.document, testRootPath);
+    localizeStyleURLs(metadata.document, metadata.document);
 
     if (!metadata.document.HasMember("metadata")) {
         mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s", paths.stylePath.c_str());
@@ -665,8 +597,7 @@ TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRo
 
     const mbgl::JSValue& metadataValue = metadata.document["metadata"];
     if (!metadataValue.HasMember("test")) {
-        mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s",
-                           paths.stylePath.c_str());
+        mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s", paths.stylePath.c_str());
         return metadata;
     }
 
@@ -694,8 +625,8 @@ TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRo
 
     if (testValue.HasMember("description")) {
         assert(testValue["description"].IsString());
-        metadata.description = std::string{ testValue["description"].GetString(),
-                                                testValue["description"].GetStringLength() };
+        metadata.description =
+            std::string{testValue["description"].GetString(), testValue["description"].GetStringLength()};
     }
 
     if (testValue.HasMember("mapMode")) {
@@ -803,7 +734,7 @@ std::string encodeBase64(const std::string& data) {
 
 std::string createResultItem(const TestMetadata& metadata, bool hasFailedTests) {
     const bool shouldHide = (hasFailedTests && metadata.status == "passed") || (metadata.status.find("ignored") != std::string::npos);
-    
+
     std::string html;
     html.append("<div class=\"test " + metadata.status + (shouldHide ? " hide" : "") + "\">\n");
     html.append(R"(<h2><span class="label" style="background: )" + metadata.color + "\">" + metadata.status + "</span> " + metadata.id + "</h2>\n");
@@ -919,21 +850,21 @@ std::string createResultPage(const TestStatistics& stats, const std::vector<Test
     return resultsPage;
 }
 
-std::string localizeURL(const std::string& url, const std::string& testRootPath) {
+std::string localizeURL(const std::string& url) {
     static const std::regex regex{"local://"};
-    if (auto vendorPath = getVendorPath(url, regex, testRootPath)) {
+    if (auto vendorPath = getVendorPath(url, regex)) {
         return *vendorPath;
     } else {
-        return getIntegrationPath(url, "", regex, testRootPath).value_or(url);
+        return getIntegrationPath(url, "", regex).value_or(url);
     }
 }
 
-void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const std::string& testRootPath) {
+void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) {
     if (root.HasMember("urls") && root["urls"].IsArray()) {
         for (auto& urlValue : root["urls"].GetArray()) {
-            const std::string path = prependFileScheme(
-                localizeMapboxTilesetURL(urlValue.GetString(), testRootPath)
-                    .value_or(localizeLocalURL(urlValue.GetString(), testRootPath).value_or(urlValue.GetString())));
+            const std::string path =
+                prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
+                                      .value_or(localizeLocalURL(urlValue.GetString()).value_or(urlValue.GetString())));
             urlValue.Set<std::string>(path, document.GetAllocator());
         }
     }
@@ -943,9 +874,9 @@ void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const s
         static const std::string video("video");
 
         mbgl::JSValue& urlValue = root["url"];
-        const std::string path = prependFileScheme(
-            localizeMapboxTilesetURL(urlValue.GetString(), testRootPath)
-                .value_or(localizeLocalURL(urlValue.GetString(), testRootPath).value_or(urlValue.GetString())));
+        const std::string path =
+            prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
+                                  .value_or(localizeLocalURL(urlValue.GetString()).value_or(urlValue.GetString())));
         urlValue.Set<std::string>(path, document.GetAllocator());
 
         if (root["type"].GetString() != image && root["type"].GetString() != video) {
@@ -964,11 +895,10 @@ void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const s
         mbgl::JSValue& tilesValue = root["tiles"];
         assert(tilesValue.IsArray());
         for (auto& tileValue : tilesValue.GetArray()) {
-            const std::string path =
-                prependFileScheme(localizeMapboxTilesURL(tileValue.GetString(), testRootPath)
-                                      .value_or(localizeLocalURL(tileValue.GetString(), testRootPath)
-                                                    .value_or(localizeHttpURL(tileValue.GetString(), testRootPath)
-                                                                  .value_or(tileValue.GetString()))));
+            const std::string path = prependFileScheme(
+                localizeMapboxTilesURL(tileValue.GetString())
+                    .value_or(localizeLocalURL(tileValue.GetString())
+                                  .value_or(localizeHttpURL(tileValue.GetString()).value_or(tileValue.GetString()))));
             tileValue.Set<std::string>(path, document.GetAllocator());
         }
     }
@@ -976,33 +906,32 @@ void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const s
     if (root.HasMember("data") && root["data"].IsString()) {
         mbgl::JSValue& dataValue = root["data"];
         const std::string path =
-            prependFileScheme(localizeLocalURL(dataValue.GetString(), testRootPath).value_or(dataValue.GetString()));
+            prependFileScheme(localizeLocalURL(dataValue.GetString()).value_or(dataValue.GetString()));
         dataValue.Set<std::string>(path, document.GetAllocator());
     }
 }
 
-void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const std::string& testRootPath) {
+void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) {
     if (root.HasMember("sources")) {
         mbgl::JSValue& sourcesValue = root["sources"];
         for (auto& sourceProperty : sourcesValue.GetObject()) {
-            localizeSourceURLs(sourceProperty.value, document, testRootPath);
+            localizeSourceURLs(sourceProperty.value, document);
         }
     }
 
     if (root.HasMember("glyphs")) {
         mbgl::JSValue& glyphsValue = root["glyphs"];
         const std::string path = prependFileScheme(
-            localizeMapboxFontsURL(glyphsValue.GetString(), testRootPath)
-                .value_or(
-                    localizeLocalURL(glyphsValue.GetString(), testRootPath, true).value_or(glyphsValue.GetString())));
+            localizeMapboxFontsURL(glyphsValue.GetString())
+                .value_or(localizeLocalURL(glyphsValue.GetString(), true).value_or(glyphsValue.GetString())));
         glyphsValue.Set<std::string>(path, document.GetAllocator());
     }
 
     if (root.HasMember("sprite")) {
         mbgl::JSValue& spriteValue = root["sprite"];
         const std::string path = prependFileScheme(
-            localizeMapboxSpriteURL(spriteValue.GetString(), testRootPath)
-                .value_or(localizeLocalURL(spriteValue.GetString(), testRootPath).value_or(spriteValue.GetString())));
+            localizeMapboxSpriteURL(spriteValue.GetString())
+                .value_or(localizeLocalURL(spriteValue.GetString()).value_or(spriteValue.GetString())));
         spriteValue.Set<std::string>(path, document.GetAllocator());
     }
 }
diff --git a/render-test/parser.hpp b/render-test/parser.hpp
index 1985b9cffc..5b3e5c7d16 100644
--- a/render-test/parser.hpp
+++ b/render-test/parser.hpp
@@ -5,14 +5,14 @@
 #include <mbgl/util/rapidjson.hpp>
 #include <mbgl/util/variant.hpp>
 
-#include <tuple>
 #include <string>
+#include <tuple>
 #include <vector>
 
 using ErrorMessage = std::string;
 using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
 
-using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::string, std::vector<TestPaths>>;
+using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::vector<TestPaths>>;
 
 JSONReply readJson(const mbgl::filesystem::path&);
 std::string serializeJsonValue(const mbgl::JSValue&);
@@ -23,21 +23,17 @@ std::vector<std::string> readExpectedJSONEntries(const mbgl::filesystem::path& b
 
 TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path);
 
-const std::string getTestPath(const std::string& rootTestPath);
-const mbgl::filesystem::path getValidPath(const std::string& basePath, const std::string& subPath);
-
 ArgumentsTuple parseArguments(int argc, char** argv);
-std::vector<std::pair<std::string, std::string>> parseIgnores(const std::string& testRootPath,
-                                                              const std::string& ignoresPath);
+std::vector<std::pair<std::string, std::string>> parseIgnores();
 
-TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRootPath);
+TestMetadata parseTestMetadata(const TestPaths& paths);
 
 std::string createResultPage(const TestStatistics&, const std::vector<TestMetadata>&, bool shuffle, uint32_t seed);
 
-std::string localizeURL(const std::string& url, const std::string& testPath);
+std::string localizeURL(const std::string& url);
 
 std::string toJSON(const mbgl::Value& value, unsigned indent, bool singleLine);
 std::string toJSON(const std::vector<mbgl::Feature>& features, unsigned indent, bool singleLine);
 
-void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const std::string& testPath);
-void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const std::string& testPath);
+void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document);
+void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document);
diff --git a/render-test/render_test.cpp b/render-test/render_test.cpp
index 2cf3f5fd65..d23ec91b08 100644
--- a/render-test/render_test.cpp
+++ b/render-test/render_test.cpp
@@ -4,6 +4,7 @@
 #include <mbgl/util/io.hpp>
 #include <mbgl/util/run_loop.hpp>
 
+#include "manifest_parser.hpp"
 #include "metadata.hpp"
 #include "parser.hpp"
 #include "runner.hpp"
@@ -43,13 +44,11 @@ int runRenderTests(int argc, char** argv) {
     bool recycleMap;
     bool shuffle;
     uint32_t seed;
-    std::string testRootPath;
-    std::string ignoresPath;
     std::vector<TestPaths> testPaths;
 
-    std::tie(recycleMap, shuffle, seed, testRootPath, ignoresPath, testPaths) = parseArguments(argc, argv);
+    std::tie(recycleMap, shuffle, seed, testPaths) = parseArguments(argc, argv);
 
-    const auto ignores = parseIgnores(testRootPath, ignoresPath);
+    const auto ignores = parseIgnores();
 
     if (shuffle) {
         printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed);
@@ -60,7 +59,7 @@ int runRenderTests(int argc, char** argv) {
     }
 
     mbgl::util::RunLoop runLoop;
-    TestRunner runner(testRootPath);
+    TestRunner runner{};
 
     std::vector<TestMetadata> metadatas;
     metadatas.reserve(testPaths.size());
@@ -68,7 +67,7 @@ int runRenderTests(int argc, char** argv) {
     TestStatistics stats;
 
     for (auto& testPath : testPaths) {
-        TestMetadata metadata = parseTestMetadata(testPath, testRootPath);
+        TestMetadata metadata = parseTestMetadata(testPath);
 
         if (!recycleMap) {
             runner.reset();
@@ -78,7 +77,7 @@ int runRenderTests(int argc, char** argv) {
         std::string& status = metadata.status;
         std::string& color = metadata.color;
 
-        const std::string::size_type rootLength = getTestPath(testRootPath).length();
+        const std::string::size_type rootLength = ManifestParser::getInstance().getTestPath().length();
         id = testPath.defaultExpectations();
         id = id.substr(rootLength + 1, id.length() - rootLength - 2);
 
@@ -138,7 +137,7 @@ int runRenderTests(int argc, char** argv) {
 
         metadatas.push_back(std::move(metadata));
     }
-
+    const auto& testRootPath = ManifestParser::getInstance().getTestPath();
     std::string resultsHTML = createResultPage(stats, metadatas, shuffle, seed);
     mbgl::util::write_file(testRootPath + "/index.html", resultsHTML);
 
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 4c0967a9e9..b41702b5d5 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -23,6 +23,7 @@
 #include <../expression-test/test_runner_common.hpp>
 #include "allocation_index.hpp"
 #include "file_source.hpp"
+#include "manifest_parser.hpp"
 #include "metadata.hpp"
 #include "parser.hpp"
 #include "runner.hpp"
@@ -101,8 +102,6 @@ std::string simpleDiff(const Value& result, const Value& expected) {
     return diff.str();
 }
 
-TestRunner::TestRunner(const std::string& testRootPath_) : maps(), testRootPath(testRootPath_) {}
-
 bool TestRunner::checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
                                        std::vector<mbgl::Feature>&& features,
                                        TestMetadata& metadata) {
@@ -487,7 +486,8 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         std::string imagePath = operationArray[2].GetString();
         imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end());
 
-        const mbgl::filesystem::path filePath = mbgl::filesystem::path(getTestPath(testRootPath)) / imagePath;
+        const mbgl::filesystem::path filePath =
+            mbgl::filesystem::path(ManifestParser::getInstance().getTestPath()) / imagePath;
 
         mbgl::optional<std::string> maybeImage = mbgl::util::readFile(filePath.string());
         if (!maybeImage) {
@@ -507,15 +507,15 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         // setStyle
         assert(operationArray.Size() >= 2u);
         if (operationArray[1].IsString()) {
-            std::string stylePath = localizeURL(operationArray[1].GetString(), testRootPath);
+            std::string stylePath = localizeURL(operationArray[1].GetString());
             auto maybeStyle = readJson(stylePath);
             if (maybeStyle.is<mbgl::JSDocument>()) {
                 auto& style = maybeStyle.get<mbgl::JSDocument>();
-                localizeStyleURLs((mbgl::JSValue&)style, style, testRootPath);
+                localizeStyleURLs((mbgl::JSValue&)style, style);
                 map.getStyle().loadJSON(serializeJsonValue(style));
             }
         } else {
-            localizeStyleURLs(operationArray[1], metadata.document, testRootPath);
+            localizeStyleURLs(operationArray[1], metadata.document);
             map.getStyle().loadJSON(serializeJsonValue(operationArray[1]));
         }
     } else if (operationArray[0].GetString() == setCenterOp) {
@@ -616,7 +616,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         assert(operationArray[1].IsString());
         assert(operationArray[2].IsObject());
 
-        localizeSourceURLs(operationArray[2], metadata.document, testRootPath);
+        localizeSourceURLs(operationArray[2], metadata.document);
 
         mbgl::style::conversion::Error error;
         auto converted = mbgl::style::conversion::convert<std::unique_ptr<mbgl::style::Source>>(operationArray[2], error, operationArray[1].GetString());
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index 3efd17bf1e..949c88af8c 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -12,7 +12,6 @@ struct TestMetadata;
 class TestRunner {
 public:
     TestRunner() = default;
-    explicit TestRunner(const std::string& testRootPath);
     bool run(TestMetadata&);
     void reset();
 
@@ -32,5 +31,4 @@ private:
         mbgl::Map map;
     };
     std::unordered_map<std::string, std::unique_ptr<Impl>> maps;
-    std::string testRootPath{TEST_RUNNER_ROOT_PATH};
 };