[render-test] Add configurable rootPath, ignorePath via CLI arguments (#15832)

* [render-test] Add configurable path via CLI arguments

* [render-test] Change ignore path config

7 files changed, 167 insertions, 109 deletions

diff --git a/next/platform/linux/linux.cmake b/next/platform/linux/linux.cmake
index 6a68a1f2d2..5bde30a61a 100644
--- a/next/platform/linux/linux.cmake
+++ b/next/platform/linux/linux.cmake
@@ -149,5 +149,9 @@ add_test(
     WORKING_DIRECTORY ${MBGL_ROOT}
 )
 
-add_test(NAME mbgl-render-test-probes COMMAND mbgl-render-test-runner tests --rootPath=render-test WORKING_DIRECTORY ${MBGL_ROOT})
+add_test(
+    NAME mbgl-render-test-probes
+    COMMAND mbgl-render-test-runner tests --rootPath=${MBGL_ROOT}/render-test
+    WORKING_DIRECTORY ${MBGL_ROOT}
+)
 add_test(NAME mbgl-query-test COMMAND mbgl-render-test-runner query-tests WORKING_DIRECTORY ${MBGL_ROOT})
diff --git a/next/platform/macos/macos.cmake b/next/platform/macos/macos.cmake
index 71e53c474a..cfabcb1cfa 100644
--- a/next/platform/macos/macos.cmake
+++ b/next/platform/macos/macos.cmake
@@ -218,8 +218,8 @@ add_test(
     COMMAND
         mbgl-render-test-runner
         tests
-        --rootPath=render-test
-        --expectationsPath=render-test/tests/mac
+        --rootPath=${MBGL_ROOT}/render-test
+        --expectationsPath=tests/mac
     WORKING_DIRECTORY ${MBGL_ROOT}
 )
 
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index d6ba561950..f410ffa860 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -96,8 +96,11 @@ std::string prependFileScheme(const std::string &url) {
     return fileScheme + url;
 }
 
-mbgl::optional<std::string> getVendorPath(const std::string& url, const std::regex& regex, bool glyphsPath = false) {
-    static const mbgl::filesystem::path vendorPath(std::string(TEST_RUNNER_ROOT_PATH) + "/vendor/");
+mbgl::optional<std::string> getVendorPath(const std::string& url,
+                                          const std::regex& regex,
+                                          const std::string& testRootPath,
+                                          bool glyphsPath = false) {
+    static const mbgl::filesystem::path vendorPath = getValidPath(testRootPath, std::string("vendor/"));
 
     mbgl::filesystem::path file = std::regex_replace(url, regex, vendorPath.string());
     if (mbgl::filesystem::exists(file.parent_path())) {
@@ -111,8 +114,13 @@ mbgl::optional<std::string> getVendorPath(const std::string& url, const std::reg
     return {};
 }
 
-mbgl::optional<std::string> getIntegrationPath(const std::string& url, const std::string& parent, const std::regex& regex, bool glyphsPath = false) {
-    static const mbgl::filesystem::path integrationPath(std::string(TEST_RUNNER_ROOT_PATH) + "/mapbox-gl-js/test/integration/");
+mbgl::optional<std::string> getIntegrationPath(const std::string& url,
+                                               const std::string& parent,
+                                               const std::regex& regex,
+                                               const std::string& testRootPath,
+                                               bool glyphsPath = false) {
+    static const mbgl::filesystem::path integrationPath =
+        getValidPath(testRootPath, std::string("mapbox-gl-js/test/integration/"));
 
     mbgl::filesystem::path file = std::regex_replace(url, regex, integrationPath.string() + parent);
     if (mbgl::filesystem::exists(file.parent_path())) {
@@ -126,46 +134,48 @@ mbgl::optional<std::string> getIntegrationPath(const std::string& url, const std
     return {};
 }
 
-mbgl::optional<std::string> localizeLocalURL(const std::string& url, bool glyphsPath = false) {
-    static const std::regex regex { "local://" };
-    if (auto vendorPath = getVendorPath(url, regex, glyphsPath)) {
+mbgl::optional<std::string> localizeLocalURL(const std::string& url,
+                                             const std::string& testRootPath,
+                                             bool glyphsPath = false) {
+    static const std::regex regex{"local://"};
+    if (auto vendorPath = getVendorPath(url, regex, testRootPath, glyphsPath)) {
         return vendorPath;
     } else {
-        return getIntegrationPath(url, "", regex, glyphsPath);
+        return getIntegrationPath(url, "", regex, testRootPath, glyphsPath);
     }
 }
 
-mbgl::optional<std::string> localizeHttpURL(const std::string& url) {
-    static const std::regex regex { "http://localhost:2900" };
-    if (auto vendorPath = getVendorPath(url, regex)) {
+mbgl::optional<std::string> localizeHttpURL(const std::string& url, const std::string& testRootPath) {
+    static const std::regex regex{"http://localhost:2900"};
+    if (auto vendorPath = getVendorPath(url, regex, testRootPath)) {
         return vendorPath;
     } else {
-        return getIntegrationPath(url, "", regex);
+        return getIntegrationPath(url, "", regex, testRootPath);
     }
 }
 
-mbgl::optional<std::string> localizeMapboxSpriteURL(const std::string& url) {
-    static const std::regex regex { "mapbox://" };
-    return getIntegrationPath(url, "", regex);
+mbgl::optional<std::string> localizeMapboxSpriteURL(const std::string& url, const std::string& testRootPath) {
+    static const std::regex regex{"mapbox://"};
+    return getIntegrationPath(url, "", regex, testRootPath);
 }
 
-mbgl::optional<std::string> localizeMapboxFontsURL(const std::string& url) {
-    static const std::regex regex { "mapbox://fonts" };
-    return getIntegrationPath(url, "glyphs/", regex, true);
+mbgl::optional<std::string> localizeMapboxFontsURL(const std::string& url, const std::string& testRootPath) {
+    static const std::regex regex{"mapbox://fonts"};
+    return getIntegrationPath(url, "glyphs/", regex, testRootPath, true);
 }
 
-mbgl::optional<std::string> localizeMapboxTilesURL(const std::string& url) {
-    static const std::regex regex { "mapbox://" };
-    if (auto vendorPath = getVendorPath(url, regex)) {
+mbgl::optional<std::string> localizeMapboxTilesURL(const std::string& url, const std::string& testRootPath) {
+    static const std::regex regex{"mapbox://"};
+    if (auto vendorPath = getVendorPath(url, regex, testRootPath)) {
         return vendorPath;
     } else {
-        return getIntegrationPath(url, "tiles/", regex);
+        return getIntegrationPath(url, "tiles/", regex, testRootPath);
     }
 }
 
-mbgl::optional<std::string> localizeMapboxTilesetURL(const std::string& url) {
-    static const std::regex regex { "mapbox://" };
-    return getIntegrationPath(url, "tilesets/", regex);
+mbgl::optional<std::string> localizeMapboxTilesetURL(const std::string& url, const std::string& testRootPath) {
+    static const std::regex regex{"mapbox://"};
+    return getIntegrationPath(url, "tilesets/", regex, testRootPath);
 }
 
 void writeJSON(rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer, const mbgl::Value& value) {
@@ -195,6 +205,33 @@ void writeJSON(rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer, const m
 
 } // namespace
 
+static const mbgl::filesystem::path DefaultRootPath{std::string(TEST_RUNNER_ROOT_PATH)};
+
+const mbgl::filesystem::path getValidPath(const std::string& basePath, const std::string& subPath) {
+    auto filePath = mbgl::filesystem::path(basePath) / subPath;
+    if (mbgl::filesystem::exists(filePath)) {
+        return filePath;
+    }
+    // Fall back to check default path
+    filePath = DefaultRootPath / subPath;
+    if (mbgl::filesystem::exists(filePath)) {
+        return filePath;
+    }
+    mbgl::Log::Warning(mbgl::Event::General, "Failed to find path: %s", subPath.c_str());
+    return mbgl::filesystem::path{};
+}
+
+/// Returns path of the render test cases directory.
+const std::string getTestPath(const std::string& rootTestPath) {
+    // Check if sub-directory exits or not
+    auto testBasePath = mbgl::filesystem::path(rootTestPath) / ("mapbox-gl-js/test/integration");
+    if (mbgl::filesystem::exists(testBasePath)) {
+        return testBasePath.string();
+    }
+    // Use root test path for further processing
+    return rootTestPath;
+}
+
 std::string toJSON(const mbgl::Value& value, unsigned indent, bool singleLine) {
     rapidjson::StringBuffer buffer;
     rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
@@ -373,17 +410,16 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
 
     args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", { 'h', "help" });
 
-    args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object",
-                              { 'r', "recycle-map" });
-    args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order",
-                           { 's', "shuffle" });
+    args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", {'r', "recycle-map"});
+    args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order", {'s', "shuffle"});
     args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)",
                                         { "seed" });
-    args::ValueFlag<std::string> testPathValue(argumentParser, "rootPath", "Test root rootPath",
-                                               { 'p', "rootPath" });
+    args::ValueFlag<std::string> testPathValue(argumentParser, "rootPath", "Test root rootPath", {'p', "rootPath"});
     args::ValueFlag<std::regex> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"});
     args::ValueFlag<std::string> expectationsPathValue(
         argumentParser, "expectationsPath", "Test expectations path", {'e', "expectationsPath"});
+    args::ValueFlag<std::string> ignoresPathValue(
+        argumentParser, "ignoresPath", "Test ignore list path", {'i', "ignoresPath"});
     args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
 
     try {
@@ -410,14 +446,16 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
         exit(3);
     }
 
-    mbgl::filesystem::path rootPath {testPathValue ? args::get(testPathValue) : TestRunner::getBasePath()};
+    const auto testRootPath = testPathValue ? args::get(testPathValue) : std::string{TEST_RUNNER_ROOT_PATH};
+    mbgl::filesystem::path rootPath{testRootPath};
     if (!mbgl::filesystem::exists(rootPath)) {
-        mbgl::Log::Error(mbgl::Event::General, "Provided rootPath '%s' does not exist.", rootPath.string().c_str());
+        mbgl::Log::Error(
+            mbgl::Event::General, "Provided test rootPath '%s' does not exist.", rootPath.string().c_str());
         exit(4);
     }
     std::vector<mbgl::filesystem::path> expectationsPaths;
     if (expectationsPathValue) {
-        auto expectationsPath = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH) / args::get(expectationsPathValue);
+        auto expectationsPath = mbgl::filesystem::path(testRootPath) / args::get(expectationsPathValue);
         if (!mbgl::filesystem::exists(expectationsPath)) {
             mbgl::Log::Error(mbgl::Event::General,
                              "Provided expectationsPath '%s' does not exist.",
@@ -427,13 +465,25 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
         expectationsPaths.emplace_back(std::move(expectationsPath));
     }
 
+    std::string ignoresPath{};
+    if (ignoresPathValue) {
+        auto path = mbgl::filesystem::path(testRootPath) / args::get(ignoresPathValue);
+        if (!mbgl::filesystem::exists(path)) {
+            mbgl::Log::Error(
+                mbgl::Event::General, "Provided ignore list path '%s' does not exist.", path.string().c_str());
+            exit(6);
+        }
+        ignoresPath = path.string();
+    }
+
     std::vector<mbgl::filesystem::path> paths;
+    auto testBasePath = mbgl::filesystem::path(getTestPath(testRootPath));
     for (const auto& id : args::get(testNameValues)) {
-        paths.emplace_back(rootPath / id);
+        paths.emplace_back(testBasePath / id);
     }
 
     if (paths.empty()) {
-        paths.emplace_back(rootPath);
+        paths.emplace_back(testBasePath);
     }
 
     // Recursively traverse through the test paths and collect test directories containing "style.json".
@@ -456,30 +506,33 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
         }
     }
 
-    return ArgumentsTuple {
-        recycleMapFlag ? args::get(recycleMapFlag) : false,
-        shuffleFlag ? args::get(shuffleFlag) : false, seedValue ? args::get(seedValue) : 1u,
-        testPathValue ? args::get(testPathValue) : TestRunner::getBasePath(),
-        std::move(testPaths)
-    };
+    return ArgumentsTuple{recycleMapFlag ? args::get(recycleMapFlag) : false,
+                          shuffleFlag ? args::get(shuffleFlag) : false,
+                          seedValue ? args::get(seedValue) : 1u,
+                          testRootPath,
+                          ignoresPath,
+                          std::move(testPaths)};
 }
 
-std::vector<std::pair<std::string, std::string>> parseIgnores() {
+std::vector<std::pair<std::string, std::string>> parseIgnores(const std::string& testRootPath,
+                                                              const std::string& ignoresPath) {
     std::vector<std::pair<std::string, std::string>> ignores;
-
-    auto mainIgnoresPath = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("platform/node/test/ignores.json");
+    auto mainIgnoresPath = getValidPath(testRootPath, "platform/node/test/ignores.json");
 
     mbgl::filesystem::path platformSpecificIgnores;
-    mbgl::filesystem::path ownTestsIgnores =
-        mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/tests/should-fail.json");
+    mbgl::filesystem::path ownTestsIgnores = getValidPath(testRootPath, "render-test/tests/should-fail.json");
 
 #ifdef __APPLE__
-    platformSpecificIgnores = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/mac-ignores.json");
+    platformSpecificIgnores = getValidPath(testRootPath, "render-test/mac-ignores.json");
 #elif __linux__
-    platformSpecificIgnores = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/linux-ignores.json");
+    platformSpecificIgnores = getValidPath(testRootPath, "render-test/linux-ignores.json");
 #endif
 
     std::vector<mbgl::filesystem::path> ignoresPaths = {mainIgnoresPath, platformSpecificIgnores, ownTestsIgnores};
+
+    if (!ignoresPath.empty()) {
+        ignoresPaths.emplace_back(ignoresPath);
+    }
     for (const auto& path : ignoresPaths) {
         auto maybeIgnores = readJson(path);
         if (!maybeIgnores.is<mbgl::JSDocument>()) {
@@ -586,7 +639,7 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
     return result;
 }
 
-TestMetadata parseTestMetadata(const TestPaths& paths) {
+TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRootPath) {
     TestMetadata metadata;
     metadata.paths = paths;
 
@@ -597,11 +650,10 @@ TestMetadata parseTestMetadata(const TestPaths& paths) {
     }
 
     metadata.document = std::move(maybeJson.get<mbgl::JSDocument>());
-    localizeStyleURLs(metadata.document, metadata.document);
+    localizeStyleURLs(metadata.document, metadata.document, testRootPath);
 
     if (!metadata.document.HasMember("metadata")) {
-        mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s",
-                           paths.stylePath.c_str());
+        mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s", paths.stylePath.c_str());
         return metadata;
     }
 
@@ -861,21 +913,21 @@ std::string createResultPage(const TestStatistics& stats, const std::vector<Test
     return resultsPage;
 }
 
-std::string localizeURL(const std::string& url) {
-    static const std::regex regex { "local://" };
-    if (auto vendorPath = getVendorPath(url, regex)) {
+std::string localizeURL(const std::string& url, const std::string& testRootPath) {
+    static const std::regex regex{"local://"};
+    if (auto vendorPath = getVendorPath(url, regex, testRootPath)) {
         return *vendorPath;
     } else {
-        return getIntegrationPath(url, "", regex).value_or(url);
+        return getIntegrationPath(url, "", regex, testRootPath).value_or(url);
     }
 }
 
-void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) {
+void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const std::string& testRootPath) {
     if (root.HasMember("urls") && root["urls"].IsArray()) {
         for (auto& urlValue : root["urls"].GetArray()) {
-            const std::string path = prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
-                .value_or(localizeLocalURL(urlValue.GetString())
-                    .value_or(urlValue.GetString())));
+            const std::string path = prependFileScheme(
+                localizeMapboxTilesetURL(urlValue.GetString(), testRootPath)
+                    .value_or(localizeLocalURL(urlValue.GetString(), testRootPath).value_or(urlValue.GetString())));
             urlValue.Set<std::string>(path, document.GetAllocator());
         }
     }
@@ -885,9 +937,9 @@ void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) {
         static const std::string video("video");
 
         mbgl::JSValue& urlValue = root["url"];
-        const std::string path = prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
-            .value_or(localizeLocalURL(urlValue.GetString())
-                .value_or(urlValue.GetString())));
+        const std::string path = prependFileScheme(
+            localizeMapboxTilesetURL(urlValue.GetString(), testRootPath)
+                .value_or(localizeLocalURL(urlValue.GetString(), testRootPath).value_or(urlValue.GetString())));
         urlValue.Set<std::string>(path, document.GetAllocator());
 
         if (root["type"].GetString() != image && root["type"].GetString() != video) {
@@ -906,43 +958,45 @@ void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) {
         mbgl::JSValue& tilesValue = root["tiles"];
         assert(tilesValue.IsArray());
         for (auto& tileValue : tilesValue.GetArray()) {
-            const std::string path = prependFileScheme(localizeMapboxTilesURL(tileValue.GetString())
-                .value_or(localizeLocalURL(tileValue.GetString())
-                    .value_or(localizeHttpURL(tileValue.GetString())
-                        .value_or(tileValue.GetString()))));
+            const std::string path =
+                prependFileScheme(localizeMapboxTilesURL(tileValue.GetString(), testRootPath)
+                                      .value_or(localizeLocalURL(tileValue.GetString(), testRootPath)
+                                                    .value_or(localizeHttpURL(tileValue.GetString(), testRootPath)
+                                                                  .value_or(tileValue.GetString()))));
             tileValue.Set<std::string>(path, document.GetAllocator());
         }
     }
 
     if (root.HasMember("data") && root["data"].IsString()) {
         mbgl::JSValue& dataValue = root["data"];
-        const std::string path = prependFileScheme(localizeLocalURL(dataValue.GetString())
-            .value_or(dataValue.GetString()));
+        const std::string path =
+            prependFileScheme(localizeLocalURL(dataValue.GetString(), testRootPath).value_or(dataValue.GetString()));
         dataValue.Set<std::string>(path, document.GetAllocator());
     }
 }
 
-void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) {
+void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const std::string& testRootPath) {
     if (root.HasMember("sources")) {
         mbgl::JSValue& sourcesValue = root["sources"];
         for (auto& sourceProperty : sourcesValue.GetObject()) {
-            localizeSourceURLs(sourceProperty.value, document);
+            localizeSourceURLs(sourceProperty.value, document, testRootPath);
         }
     }
 
     if (root.HasMember("glyphs")) {
         mbgl::JSValue& glyphsValue = root["glyphs"];
-        const std::string path = prependFileScheme(localizeMapboxFontsURL(glyphsValue.GetString())
-            .value_or(localizeLocalURL(glyphsValue.GetString(), true)
-                .value_or(glyphsValue.GetString())));
+        const std::string path = prependFileScheme(
+            localizeMapboxFontsURL(glyphsValue.GetString(), testRootPath)
+                .value_or(
+                    localizeLocalURL(glyphsValue.GetString(), testRootPath, true).value_or(glyphsValue.GetString())));
         glyphsValue.Set<std::string>(path, document.GetAllocator());
     }
 
     if (root.HasMember("sprite")) {
         mbgl::JSValue& spriteValue = root["sprite"];
-        const std::string path = prependFileScheme(localizeMapboxSpriteURL(spriteValue.GetString())
-            .value_or(localizeLocalURL(spriteValue.GetString())
-                .value_or(spriteValue.GetString())));
+        const std::string path = prependFileScheme(
+            localizeMapboxSpriteURL(spriteValue.GetString(), testRootPath)
+                .value_or(localizeLocalURL(spriteValue.GetString(), testRootPath).value_or(spriteValue.GetString())));
         spriteValue.Set<std::string>(path, document.GetAllocator());
     }
 }
diff --git a/render-test/parser.hpp b/render-test/parser.hpp
index 3c857b7e1e..1985b9cffc 100644
--- a/render-test/parser.hpp
+++ b/render-test/parser.hpp
@@ -12,7 +12,7 @@
 using ErrorMessage = std::string;
 using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
 
-using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector<TestPaths>>;
+using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::string, std::vector<TestPaths>>;
 
 JSONReply readJson(const mbgl::filesystem::path&);
 std::string serializeJsonValue(const mbgl::JSValue&);
@@ -23,17 +23,21 @@ std::vector<std::string> readExpectedJSONEntries(const mbgl::filesystem::path& b
 
 TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path);
 
+const std::string getTestPath(const std::string& rootTestPath);
+const mbgl::filesystem::path getValidPath(const std::string& basePath, const std::string& subPath);
+
 ArgumentsTuple parseArguments(int argc, char** argv);
-std::vector<std::pair<std::string, std::string>> parseIgnores();
+std::vector<std::pair<std::string, std::string>> parseIgnores(const std::string& testRootPath,
+                                                              const std::string& ignoresPath);
 
-TestMetadata parseTestMetadata(const TestPaths& paths);
+TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRootPath);
 
 std::string createResultPage(const TestStatistics&, const std::vector<TestMetadata>&, bool shuffle, uint32_t seed);
 
-std::string localizeURL(const std::string& url);
+std::string localizeURL(const std::string& url, const std::string& testPath);
 
 std::string toJSON(const mbgl::Value& value, unsigned indent, bool singleLine);
 std::string toJSON(const std::vector<mbgl::Feature>& features, unsigned indent, bool singleLine);
 
-void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document);
-void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document);
\ No newline at end of file
+void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const std::string& testPath);
+void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const std::string& testPath);
diff --git a/render-test/render_test.cpp b/render-test/render_test.cpp
index df1658265a..2cf3f5fd65 100644
--- a/render-test/render_test.cpp
+++ b/render-test/render_test.cpp
@@ -44,12 +44,12 @@ int runRenderTests(int argc, char** argv) {
     bool shuffle;
     uint32_t seed;
     std::string testRootPath;
+    std::string ignoresPath;
     std::vector<TestPaths> testPaths;
 
-    std::tie(recycleMap, shuffle, seed, testRootPath, testPaths) = parseArguments(argc, argv);
-    const std::string::size_type rootLength = testRootPath.length();
+    std::tie(recycleMap, shuffle, seed, testRootPath, ignoresPath, testPaths) = parseArguments(argc, argv);
 
-    const auto ignores = parseIgnores();
+    const auto ignores = parseIgnores(testRootPath, ignoresPath);
 
     if (shuffle) {
         printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed);
@@ -60,7 +60,7 @@ int runRenderTests(int argc, char** argv) {
     }
 
     mbgl::util::RunLoop runLoop;
-    TestRunner runner;
+    TestRunner runner(testRootPath);
 
     std::vector<TestMetadata> metadatas;
     metadatas.reserve(testPaths.size());
@@ -68,7 +68,7 @@ int runRenderTests(int argc, char** argv) {
     TestStatistics stats;
 
     for (auto& testPath : testPaths) {
-        TestMetadata metadata = parseTestMetadata(testPath);
+        TestMetadata metadata = parseTestMetadata(testPath, testRootPath);
 
         if (!recycleMap) {
             runner.reset();
@@ -78,6 +78,7 @@ int runRenderTests(int argc, char** argv) {
         std::string& status = metadata.status;
         std::string& color = metadata.color;
 
+        const std::string::size_type rootLength = getTestPath(testRootPath).length();
         id = testPath.defaultExpectations();
         id = id.substr(rootLength + 1, id.length() - rootLength - 2);
 
@@ -166,4 +167,4 @@ int runRenderTests(int argc, char** argv) {
     return stats.failedTests + stats.erroredTests == 0 ? 0 : 1;
 }
 
-} // namespace mbgl
\ No newline at end of file
+} // namespace mbgl
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 46b890b76c..4c0967a9e9 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -58,12 +58,6 @@ public:
 };
 
 // static
-const std::string& TestRunner::getBasePath() {
-    const static std::string result = std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration");
-    return result;
-}
-
-// static
 gfx::HeadlessBackend::SwapBehaviour swapBehavior(MapMode mode) {
     return mode == MapMode::Continuous ? gfx::HeadlessBackend::SwapBehaviour::Flush
                                        : gfx::HeadlessBackend::SwapBehaviour::NoFlush;
@@ -107,6 +101,8 @@ std::string simpleDiff(const Value& result, const Value& expected) {
     return diff.str();
 }
 
+TestRunner::TestRunner(const std::string& testRootPath_) : maps(), testRootPath(testRootPath_) {}
+
 bool TestRunner::checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
                                        std::vector<mbgl::Feature>&& features,
                                        TestMetadata& metadata) {
@@ -491,7 +487,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         std::string imagePath = operationArray[2].GetString();
         imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end());
 
-        const mbgl::filesystem::path filePath(std::string(TEST_RUNNER_ROOT_PATH) + "/mapbox-gl-js/test/integration/" + imagePath);
+        const mbgl::filesystem::path filePath = mbgl::filesystem::path(getTestPath(testRootPath)) / imagePath;
 
         mbgl::optional<std::string> maybeImage = mbgl::util::readFile(filePath.string());
         if (!maybeImage) {
@@ -511,15 +507,15 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         // setStyle
         assert(operationArray.Size() >= 2u);
         if (operationArray[1].IsString()) {
-            std::string stylePath = localizeURL(operationArray[1].GetString());
+            std::string stylePath = localizeURL(operationArray[1].GetString(), testRootPath);
             auto maybeStyle = readJson(stylePath);
             if (maybeStyle.is<mbgl::JSDocument>()) {
                 auto& style = maybeStyle.get<mbgl::JSDocument>();
-                localizeStyleURLs((mbgl::JSValue&)style, style);
+                localizeStyleURLs((mbgl::JSValue&)style, style, testRootPath);
                 map.getStyle().loadJSON(serializeJsonValue(style));
             }
         } else {
-            localizeStyleURLs(operationArray[1], metadata.document);
+            localizeStyleURLs(operationArray[1], metadata.document, testRootPath);
             map.getStyle().loadJSON(serializeJsonValue(operationArray[1]));
         }
     } else if (operationArray[0].GetString() == setCenterOp) {
@@ -620,7 +616,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         assert(operationArray[1].IsString());
         assert(operationArray[2].IsObject());
 
-        localizeSourceURLs(operationArray[2], metadata.document);
+        localizeSourceURLs(operationArray[2], metadata.document, testRootPath);
 
         mbgl::style::conversion::Error error;
         auto converted = mbgl::style::conversion::convert<std::unique_ptr<mbgl::style::Source>>(operationArray[2], error, operationArray[1].GetString());
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index 6aba2d2391..3efd17bf1e 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -4,6 +4,7 @@
 #include <mbgl/map/map.hpp>
 
 #include <memory>
+#include <string>
 
 class TestRunnerMapObserver;
 struct TestMetadata;
@@ -11,13 +12,10 @@ struct TestMetadata;
 class TestRunner {
 public:
     TestRunner() = default;
-
+    explicit TestRunner(const std::string& testRootPath);
     bool run(TestMetadata&);
     void reset();
 
-    /// Returns path of the render tests root directory.
-    static const std::string& getBasePath();
-
 private:
     bool runOperations(const std::string& key, TestMetadata&);
     bool checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
@@ -34,4 +32,5 @@ private:
         mbgl::Map map;
     };
     std::unordered_map<std::string, std::unique_ptr<Impl>> maps;
-};
\ No newline at end of file
+    std::string testRootPath{TEST_RUNNER_ROOT_PATH};
+};