[render-test] Refactory manifest parser

11 files changed, 390 insertions, 380 deletions

diff --git a/circle.yml b/circle.yml
index 103ea8c685..a109ef2b41 100644
--- a/circle.yml
+++ b/circle.yml
@@ -267,9 +267,7 @@ commands:
           when: on_fail
           command: |
             mkdir -p /tmp/tests/render
-            mkdir -p /tmp/tests/probe
-            if [ -f index.html ]; then cp index.html /tmp/tests/render; fi
-            if [ -f render-test/index.html ]; then cp render-test/index.html /tmp/tests/probe; fi
+            if [ -f render-test/index.html ]; then cp render-test/index.html /tmp/tests/render; fi
             mkdir -p /tmp/tests/coredumps
             if ls core* 1> /dev/null 2>&1; then cp core* /tmp/tests/coredumps; fi
       - store_artifacts:
diff --git a/render-test/android-manifest.json b/render-test/android-manifest.json
index 5144c747d1..dcd013eee5 100644
--- a/render-test/android-manifest.json
+++ b/render-test/android-manifest.json
@@ -1,6 +1,6 @@
 {
     "base_test_path":"mapbox-gl-js/test/integration",
-    "probe_test_path":"",
+    "probe_test_path":".",
     "expectation_paths":["render-test/expected/render-tests"],
     "ignore_paths":["platform/node/test/ignores.json", "render-test/linux-ignores.json", "render-test/tests/should-fail.json"],
     "vendor_path":"vendor",
diff --git a/render-test/linux-manifest.json b/render-test/linux-manifest.json
index 4cb2a699b4..e0b8e101d8 100644
--- a/render-test/linux-manifest.json
+++ b/render-test/linux-manifest.json
@@ -1,6 +1,6 @@
 {
     "base_test_path":"../mapbox-gl-js/test/integration",
-    "probe_test_path":"",
+    "probe_test_path":".",
     "expectation_paths":["expected/render-tests"],
     "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json", "../render-test/tests/should-fail.json"],
     "vendor_path":"../vendor",
diff --git a/render-test/mac-manifest.json b/render-test/mac-manifest.json
index c54cc958af..b53f841506 100644
--- a/render-test/mac-manifest.json
+++ b/render-test/mac-manifest.json
@@ -1,6 +1,6 @@
 {
     "base_test_path":"../mapbox-gl-js/test/integration",
-    "probe_test_path":"",
+    "probe_test_path":".",
     "expectation_paths":["expected/render-tests", "tests/mac"],
     "ignore_paths":["../platform/node/test/ignores.json", "../render-test/mac-ignores.json", "../render-test/tests/should-fail.json"],
     "vendor_path":"../vendor",
diff --git a/render-test/manifest_parser.cpp b/render-test/manifest_parser.cpp
index 458df137c8..f7398a9856 100644
--- a/render-test/manifest_parser.cpp
+++ b/render-test/manifest_parser.cpp
@@ -1,81 +1,283 @@
 #include "manifest_parser.hpp"
-#include <mbgl/util/logging.hpp>
-#include <mbgl/util/rapidjson.hpp>
+#include "filesystem.hpp"
 #include "parser.hpp"
 
-mbgl::filesystem::path ManifestParser::getValidPath(const std::string& path) {
-    const static mbgl::filesystem::path BasePath{rootPath};
-    mbgl::filesystem::path result{path};
-    if (result.is_relative()) {
-        result = BasePath / result;
+#include <mbgl/util/logging.hpp>
+
+#include <random>
+
+namespace {
+std::string removeURLArguments(const std::string& url) {
+    std::string::size_type index = url.find('?');
+    if (index != std::string::npos) {
+        return url.substr(0, index);
     }
-    if (mbgl::filesystem::exists(result)) {
-        return result;
+    return url;
+}
+
+std::string prependFileScheme(const std::string& url) {
+    static const std::string fileScheme("file://");
+    return fileScheme + url;
+}
+} // namespace
+
+const std::vector<TestPaths>& Manifest::getTestPaths() const {
+    return testPaths;
+}
+const std::vector<std::pair<std::string, std::string>>& Manifest::getIgnores() const {
+    return ignores;
+}
+const std::string& Manifest::getTestRootPath() const {
+    return testRootPath;
+}
+
+void Manifest::doShuffle(uint32_t seed) {
+    std::seed_seq sequence{seed};
+    std::mt19937 shuffler(sequence);
+    std::shuffle(testPaths.begin(), testPaths.end(), shuffler);
+}
+
+std::string Manifest::localizeURL(const std::string& url) const {
+    static const std::regex regex{"local://"};
+    if (auto vendorPath = getVendorPath(url, regex)) {
+        return *vendorPath;
+    } else {
+        return getIntegrationPath(url, "", regex).value_or(url);
+    }
+}
+
+void Manifest::localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const {
+    if (root.HasMember("urls") && root["urls"].IsArray()) {
+        for (auto& urlValue : root["urls"].GetArray()) {
+            const std::string path =
+                prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
+                                      .value_or(localizeLocalURL(urlValue.GetString()).value_or(urlValue.GetString())));
+            urlValue.Set<std::string>(path, document.GetAllocator());
+        }
+    }
+
+    if (root.HasMember("url")) {
+        static const std::string image("image");
+        static const std::string video("video");
+
+        mbgl::JSValue& urlValue = root["url"];
+        const std::string path =
+            prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
+                                  .value_or(localizeLocalURL(urlValue.GetString()).value_or(urlValue.GetString())));
+        urlValue.Set<std::string>(path, document.GetAllocator());
+
+        if (root["type"].GetString() != image && root["type"].GetString() != video) {
+            const auto tilesetPath = std::string(urlValue.GetString()).erase(0u, 7u); // remove "file://"
+            auto maybeTileset = readJson(tilesetPath);
+            if (maybeTileset.is<mbgl::JSDocument>()) {
+                const auto& tileset = maybeTileset.get<mbgl::JSDocument>();
+                assert(tileset.HasMember("tiles"));
+                root.AddMember("tiles", (mbgl::JSValue&)tileset["tiles"], document.GetAllocator());
+                root.RemoveMember("url");
+            }
+        }
+    }
+
+    if (root.HasMember("tiles")) {
+        mbgl::JSValue& tilesValue = root["tiles"];
+        assert(tilesValue.IsArray());
+        for (auto& tileValue : tilesValue.GetArray()) {
+            const std::string path = prependFileScheme(
+                localizeMapboxTilesURL(tileValue.GetString())
+                    .value_or(localizeLocalURL(tileValue.GetString())
+                                  .value_or(localizeHttpURL(tileValue.GetString()).value_or(tileValue.GetString()))));
+            tileValue.Set<std::string>(path, document.GetAllocator());
+        }
+    }
+
+    if (root.HasMember("data") && root["data"].IsString()) {
+        mbgl::JSValue& dataValue = root["data"];
+        const std::string path =
+            prependFileScheme(localizeLocalURL(dataValue.GetString()).value_or(dataValue.GetString()));
+        dataValue.Set<std::string>(path, document.GetAllocator());
+    }
+}
+
+void Manifest::localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const {
+    if (root.HasMember("sources")) {
+        mbgl::JSValue& sourcesValue = root["sources"];
+        for (auto& sourceProperty : sourcesValue.GetObject()) {
+            localizeSourceURLs(sourceProperty.value, document);
+        }
+    }
+
+    if (root.HasMember("glyphs")) {
+        mbgl::JSValue& glyphsValue = root["glyphs"];
+        const std::string path = prependFileScheme(
+            localizeMapboxFontsURL(glyphsValue.GetString())
+                .value_or(localizeLocalURL(glyphsValue.GetString(), true).value_or(glyphsValue.GetString())));
+        glyphsValue.Set<std::string>(path, document.GetAllocator());
+    }
+
+    if (root.HasMember("sprite")) {
+        mbgl::JSValue& spriteValue = root["sprite"];
+        const std::string path = prependFileScheme(
+            localizeMapboxSpriteURL(spriteValue.GetString())
+                .value_or(localizeLocalURL(spriteValue.GetString()).value_or(spriteValue.GetString())));
+        spriteValue.Set<std::string>(path, document.GetAllocator());
+    }
+}
+
+mbgl::optional<std::string> Manifest::localizeLocalURL(const std::string& url, bool glyphsPath) const {
+    static const std::regex regex{"local://"};
+    if (auto vendorPath = getVendorPath(url, regex, glyphsPath)) {
+        return vendorPath;
+    } else {
+        return getIntegrationPath(url, "", regex, glyphsPath);
+    }
+}
+
+mbgl::optional<std::string> Manifest::localizeHttpURL(const std::string& url) const {
+    static const std::regex regex{"http://localhost:2900"};
+    if (auto vendorPath = getVendorPath(url, regex)) {
+        return vendorPath;
+    } else {
+        return getIntegrationPath(url, "", regex);
     }
-    mbgl::Log::Warning(mbgl::Event::General, "Invalid path is provoided inside the manifest file: %s", path.c_str());
-    return mbgl::filesystem::path{};
 }
 
-const std::string ManifestParser::getVendorPath() const {
-    return vendorPath;
+mbgl::optional<std::string> Manifest::localizeMapboxSpriteURL(const std::string& url) const {
+    static const std::regex regex{"mapbox://"};
+    return getIntegrationPath(url, "", regex);
 }
-const std::string ManifestParser::getIntegrationPath() const {
-    return assetPath;
+
+mbgl::optional<std::string> Manifest::localizeMapboxFontsURL(const std::string& url) const {
+    static const std::regex regex{"mapbox://fonts"};
+    return getIntegrationPath(url, "glyphs/", regex, true);
 }
-const mbgl::filesystem::path ManifestParser::getBaseTestPath() const {
-    return baseTestPath;
+
+mbgl::optional<std::string> Manifest::localizeMapboxTilesURL(const std::string& url) const {
+    static const std::regex regex{"mapbox://"};
+    if (auto vendorPath = getVendorPath(url, regex)) {
+        return vendorPath;
+    } else {
+        return getIntegrationPath(url, "tiles/", regex);
+    }
 }
-const mbgl::filesystem::path ManifestParser::getProbeTestPath() const {
-    return probeTestPath;
+
+mbgl::optional<std::string> Manifest::localizeMapboxTilesetURL(const std::string& url) const {
+    static const std::regex regex{"mapbox://"};
+    return getIntegrationPath(url, "tilesets/", regex);
 }
-const std::vector<mbgl::filesystem::path> ManifestParser::getExpectationsPaths() const {
-    return expectationPaths;
+
+mbgl::optional<std::string> Manifest::getVendorPath(const std::string& url,
+                                                    const std::regex& regex,
+                                                    bool glyphsPath) const {
+    mbgl::filesystem::path file = std::regex_replace(url, regex, vendorPath);
+    if (mbgl::filesystem::exists(file.parent_path())) {
+        return removeURLArguments(file.string());
+    }
+
+    if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
+        return removeURLArguments(file.string());
+    }
+
+    return {};
 }
-const std::vector<mbgl::filesystem::path> ManifestParser::getIgnoresPaths() const {
-    return ignorePaths;
+
+mbgl::optional<std::string> Manifest::getIntegrationPath(const std::string& url,
+                                                         const std::string& parent,
+                                                         const std::regex& regex,
+                                                         bool glyphsPath) const {
+    mbgl::filesystem::path file = std::regex_replace(url, regex, assetPath + parent);
+    if (mbgl::filesystem::exists(file.parent_path())) {
+        return removeURLArguments(file.string());
+    }
+
+    if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
+        return removeURLArguments(file.string());
+    }
+
+    return {};
+}
+
+namespace {
+std::vector<std::pair<std::string, std::string>> parseIgnores(const std::vector<mbgl::filesystem::path>& ignoresPaths) {
+    std::vector<std::pair<std::string, std::string>> ignores;
+    for (const auto& path : ignoresPaths) {
+        auto maybeIgnores = readJson(path);
+        if (!maybeIgnores.is<mbgl::JSDocument>()) {
+            continue;
+        }
+        for (const auto& property : maybeIgnores.get<mbgl::JSDocument>().GetObject()) {
+            const std::string ignore = {property.name.GetString(), property.name.GetStringLength()};
+            const std::string reason = {property.value.GetString(), property.value.GetStringLength()};
+            ignores.emplace_back(std::make_pair(ignore, reason));
+        }
+    }
+
+    return ignores;
 }
-const std::string ManifestParser::getTestPath() const {
-    assert(!testPath.empty());
-    return testPath;
+
+std::vector<mbgl::filesystem::path> getTestExpectations(mbgl::filesystem::path testPath,
+                                                        const mbgl::filesystem::path& testsRootPath,
+                                                        std::vector<mbgl::filesystem::path> expectationsPaths) {
+    std::vector<mbgl::filesystem::path> expectations{std::move(testPath.remove_filename())};
+    const auto& defaultTestExpectationsPath = expectations.front().string();
+
+    const std::regex regex{testsRootPath.string()};
+    for (const auto& path : expectationsPaths) {
+        expectations.emplace_back(std::regex_replace(defaultTestExpectationsPath, regex, path.string()));
+        assert(!expectations.back().empty());
+    }
+
+    return expectations;
 }
-void ManifestParser::setTestPath(const std::string& testPath_) {
-    testPath = testPath_;
-    if (testPath.back() == '/') {
-        testPath.pop_back();
+
+mbgl::filesystem::path getValidPath(const std::string& rootPath, const std::string& path) {
+    const static mbgl::filesystem::path BasePath{rootPath};
+    mbgl::filesystem::path result{path};
+    if (result.is_relative()) {
+        result = BasePath / result;
     }
+    if (mbgl::filesystem::exists(result)) {
+        return result;
+    }
+    mbgl::Log::Warning(mbgl::Event::General, "Invalid path is provoided inside the manifest file: %s", path.c_str());
+    return mbgl::filesystem::path{};
 }
 
-bool ManifestParser::parseManifest(const std::string& rootPath_) {
-    rootPath = rootPath_;
-    auto filePath = rootPath;
+} // namespace
+
+mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& rootPath,
+                                                       const std::vector<std::string>& testNames,
+                                                       const std::string& testFilter) {
+    static Manifest manifest;
+    auto filePath = mbgl::filesystem::path(rootPath);
 #ifdef __APPLE__
-    filePath += "/mac-manifest.json";
+    filePath /= "mac-manifest.json";
 #elif __ANDROID__
-    filePath += "/android-manifest.json";
+    filePath /= "android-manifest.json";
 #else
-    filePath += "/linux-manifest.json";
+    filePath /= "linux-manifest.json";
 #endif
     if (!mbgl::filesystem::exists(filePath)) {
         mbgl::Log::Error(mbgl::Event::General, "Provided manifest file: %s is not valid", filePath.c_str());
-        return false;
+        return {};
     }
     auto contents = readJson(filePath);
     if (!contents.is<mbgl::JSDocument>()) {
         mbgl::Log::Error(mbgl::Event::General, "Provided manifest file: %s is not a valid json", filePath.c_str());
-        return false;
+        return {};
     }
+
     auto document = std::move(contents.get<mbgl::JSDocument>());
     if (document.HasMember("asset_path")) {
         const auto& assetPathValue = document["asset_path"];
         if (!assetPathValue.IsString()) {
             mbgl::Log::Warning(
                 mbgl::Event::General, "Invalid assetPath is provoided inside the manifest file: %s", filePath.c_str());
-            return false;
+            return {};
         }
 
-        assetPath = getValidPath(assetPathValue.GetString()).string();
-        if (assetPath.back() != '/') {
-            assetPath.push_back('/');
+        manifest.assetPath = getValidPath(rootPath, assetPathValue.GetString()).string();
+        if (manifest.assetPath.back() != '/') {
+            manifest.assetPath.push_back('/');
         }
     }
     if (document.HasMember("vendor_path")) {
@@ -83,76 +285,119 @@ bool ManifestParser::parseManifest(const std::string& rootPath_) {
         if (!vendorPathValue.IsString()) {
             mbgl::Log::Warning(
                 mbgl::Event::General, "Invalid vendorPath is provoided inside the manifest file: %s", filePath.c_str());
-            return false;
+            return {};
         }
         auto path = std::string(vendorPathValue.GetString());
 
-        vendorPath = getValidPath(vendorPathValue.GetString()).string();
-        if (vendorPath.back() != '/') {
-            vendorPath.push_back('/');
+        manifest.vendorPath = getValidPath(rootPath, vendorPathValue.GetString()).string();
+        if (manifest.vendorPath.back() != '/') {
+            manifest.vendorPath.push_back('/');
         }
     }
+    mbgl::filesystem::path baseTestPath;
     if (document.HasMember("base_test_path")) {
         const auto& testPathValue = document["base_test_path"];
         if (!testPathValue.IsString()) {
             mbgl::Log::Warning(
                 mbgl::Event::General, "Invalid testPath is provoided inside the manifest file: %s", filePath.c_str());
-            return false;
+            return {};
         }
-        auto path = getValidPath(testPathValue.GetString()).string();
+        auto path = getValidPath(rootPath, testPathValue.GetString()).string();
         if (path.back() == '/') {
             path.pop_back();
         }
         baseTestPath = mbgl::filesystem::path(path);
-        setTestPath(baseTestPath.string());
     }
+    mbgl::filesystem::path probeTestPath;
     if (document.HasMember("probe_test_path")) {
         const auto& testPathValue = document["probe_test_path"];
         if (!testPathValue.IsString()) {
             mbgl::Log::Warning(
                 mbgl::Event::General, "Invalid testPath is provoided inside the manifest file: %s", filePath.c_str());
-            return false;
+            return {};
         }
-        auto path = getValidPath(testPathValue.GetString()).string();
+        auto path = getValidPath(rootPath, testPathValue.GetString()).string();
         if (path.back() == '/') {
             path.pop_back();
         }
         probeTestPath = mbgl::filesystem::path(path);
     }
+    std::vector<mbgl::filesystem::path> expectationPaths{};
     if (document.HasMember("expectation_paths")) {
         const auto& expectationPathValue = document["expectation_paths"];
         if (!expectationPathValue.IsArray()) {
             mbgl::Log::Warning(mbgl::Event::General,
                                "Invalid expectationPath is provoided inside the manifest file: %s",
                                filePath.c_str());
-            return false;
+            return {};
         }
         for (const auto& value : expectationPathValue.GetArray()) {
             if (!value.IsString()) {
-                return false;
+                return {};
             }
-            expectationPaths.emplace_back(getValidPath(value.GetString()));
+            expectationPaths.emplace_back(getValidPath(rootPath, value.GetString()));
             if (expectationPaths.back().empty()) {
-                return false;
+                return {};
             }
         }
     }
+    std::vector<mbgl::filesystem::path> ignorePaths{};
     if (document.HasMember("ignore_paths")) {
         const auto& ignorePathValue = document["ignore_paths"];
         if (!ignorePathValue.IsArray()) {
             mbgl::Log::Warning(
                 mbgl::Event::General, "Invalid ignorePath is provoided inside the manifest file: %s", filePath.c_str());
-            return false;
+            return {};
         }
         for (const auto& value : ignorePathValue.GetArray()) {
             if (!value.IsString()) {
-                return false;
+                return {};
             }
-            ignorePaths.emplace_back(getValidPath(value.GetString()));
+            ignorePaths.emplace_back(getValidPath(rootPath, value.GetString()));
             if (ignorePaths.back().empty()) {
-                return false;
+                return {};
             }
         }
+        manifest.ignores = parseIgnores(ignorePaths);
+    }
+    bool enbaleProbeTest{false};
+    std::vector<mbgl::filesystem::path> paths;
+    for (const auto& id : testNames) {
+        if (id == "tests") {
+            paths.emplace_back(probeTestPath / id);
+            enbaleProbeTest = true;
+        } else {
+            paths.emplace_back(baseTestPath / id);
+        }
+    }
+    if (paths.empty()) {
+        paths.emplace_back(baseTestPath);
+    }
+
+    // Recursively traverse through the test paths and collect test directories containing "style.son".
+    auto& testPaths = manifest.testPaths;
+    testPaths.reserve(paths.size());
+    for (const auto& path : paths) {
+        if (!mbgl::filesystem::exists(path)) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Provided test folder '%s' does not exist.", path.string().c_str());
+            continue;
+        }
+        for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(path)) {
+            // Skip paths that fail regexp match.
+            if (!testFilter.empty() && !std::regex_match(testPath.path().string(), std::regex(testFilter))) {
+                continue;
+            }
+
+            if (testPath.path().filename() == "style.json") {
+                testPaths.emplace_back(testPath, getTestExpectations(testPath, path, expectationPaths));
+            }
+        }
+    }
+
+    manifest.testRootPath = enbaleProbeTest ? probeTestPath.string() : baseTestPath.string();
+    if (manifest.testRootPath.back() == '/') {
+        manifest.testRootPath.pop_back();
     }
-    return true;
+    return mbgl::optional<Manifest>(manifest);
 }
diff --git a/render-test/manifest_parser.hpp b/render-test/manifest_parser.hpp
index fd7b81d9c8..7c96a6e6c4 100644
--- a/render-test/manifest_parser.hpp
+++ b/render-test/manifest_parser.hpp
@@ -1,35 +1,51 @@
 #pragma once
 
+#include "metadata.hpp"
+
+#include <mbgl/util/optional.hpp>
+#include <mbgl/util/rapidjson.hpp>
+
+#include <regex>
 #include <string>
+#include <utility>
 #include <vector>
-#include "filesystem.hpp"
 
-class ManifestParser {
+struct Manifest {
 public:
-    static ManifestParser& getInstance() {
-        static ManifestParser instance;
-        return instance;
-    }
-    const std::string getVendorPath() const;
-    const std::string getIntegrationPath() const;
-    const mbgl::filesystem::path getBaseTestPath() const;
-    const mbgl::filesystem::path getProbeTestPath() const;
-    const std::vector<mbgl::filesystem::path> getExpectationsPaths() const;
-    const std::vector<mbgl::filesystem::path> getIgnoresPaths() const;
-    bool parseManifest(const std::string& rootPath_);
-    const std::string getTestPath() const;
-    void setTestPath(const std::string& testPath_);
+    const std::vector<std::pair<std::string, std::string>>& getIgnores() const;
+    const std::vector<TestPaths>& getTestPaths() const;
+    const std::string& getTestRootPath() const;
+    void doShuffle(uint32_t seed);
 
-private:
-    ManifestParser() = default;
-    mbgl::filesystem::path getValidPath(const std::string& path);
+    std::string localizeURL(const std::string& url) const;
+    void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const;
+    void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const;
 
-    std::string rootPath{};
-    std::string testPath{};
+private:
+    friend class ManifestParser;
+    mbgl::optional<std::string> localizeLocalURL(const std::string& url, bool glyphsPath = false) const;
+    mbgl::optional<std::string> localizeHttpURL(const std::string& url) const;
+    mbgl::optional<std::string> localizeMapboxSpriteURL(const std::string& url) const;
+    mbgl::optional<std::string> localizeMapboxFontsURL(const std::string& url) const;
+    mbgl::optional<std::string> localizeMapboxTilesURL(const std::string& url) const;
+    mbgl::optional<std::string> localizeMapboxTilesetURL(const std::string& url) const;
+    mbgl::optional<std::string> getVendorPath(const std::string& url,
+                                              const std::regex& regex,
+                                              bool glyphsPath = false) const;
+    mbgl::optional<std::string> getIntegrationPath(const std::string& url,
+                                                   const std::string& parent,
+                                                   const std::regex& regex,
+                                                   bool glyphsPath = false) const;
+    std::string testRootPath{};
     std::string vendorPath{};
     std::string assetPath{};
-    mbgl::filesystem::path baseTestPath{};
-    mbgl::filesystem::path probeTestPath{};
-    std::vector<mbgl::filesystem::path> expectationPaths{};
-    std::vector<mbgl::filesystem::path> ignorePaths{};
+    std::vector<std::pair<std::string, std::string>> ignores{};
+    std::vector<TestPaths> testPaths{};
+};
+
+class ManifestParser {
+public:
+    static mbgl::optional<Manifest> parseManifest(const std::string& rootPath_,
+                                                  const std::vector<std::string>& testNames,
+                                                  const std::string& testFilter);
 };
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index c56fc7ca19..b3ace13d5c 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -84,91 +84,6 @@ const char* resultsHeaderButtons = R"HTML(
 </h1>
 )HTML";
 
-std::string removeURLArguments(const std::string &url) {
-    std::string::size_type index = url.find('?');
-    if (index != std::string::npos) {
-        return url.substr(0, index);
-    }
-    return url;
-}
-
-std::string prependFileScheme(const std::string &url) {
-    static const std::string fileScheme("file://");
-    return fileScheme + url;
-}
-
-mbgl::optional<std::string> getVendorPath(const std::string& url, const std::regex& regex, bool glyphsPath = false) {
-    mbgl::filesystem::path file = std::regex_replace(url, regex, ManifestParser::getInstance().getVendorPath());
-    if (mbgl::filesystem::exists(file.parent_path())) {
-        return removeURLArguments(file.string());
-    }
-
-    if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
-        return removeURLArguments(file.string());
-    }
-
-    return {};
-}
-
-mbgl::optional<std::string> getIntegrationPath(const std::string& url,
-                                               const std::string& parent,
-                                               const std::regex& regex,
-                                               bool glyphsPath = false) {
-    mbgl::filesystem::path file =
-        std::regex_replace(url, regex, ManifestParser::getInstance().getIntegrationPath() + parent);
-    if (mbgl::filesystem::exists(file.parent_path())) {
-        return removeURLArguments(file.string());
-    }
-
-    if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
-        return removeURLArguments(file.string());
-    }
-
-    return {};
-}
-
-mbgl::optional<std::string> localizeLocalURL(const std::string& url, bool glyphsPath = false) {
-    static const std::regex regex{"local://"};
-    if (auto vendorPath = getVendorPath(url, regex, glyphsPath)) {
-        return vendorPath;
-    } else {
-        return getIntegrationPath(url, "", regex, glyphsPath);
-    }
-}
-
-mbgl::optional<std::string> localizeHttpURL(const std::string& url) {
-    static const std::regex regex{"http://localhost:2900"};
-    if (auto vendorPath = getVendorPath(url, regex)) {
-        return vendorPath;
-    } else {
-        return getIntegrationPath(url, "", regex);
-    }
-}
-
-mbgl::optional<std::string> localizeMapboxSpriteURL(const std::string& url) {
-    static const std::regex regex{"mapbox://"};
-    return getIntegrationPath(url, "", regex);
-}
-
-mbgl::optional<std::string> localizeMapboxFontsURL(const std::string& url) {
-    static const std::regex regex{"mapbox://fonts"};
-    return getIntegrationPath(url, "glyphs/", regex, true);
-}
-
-mbgl::optional<std::string> localizeMapboxTilesURL(const std::string& url) {
-    static const std::regex regex{"mapbox://"};
-    if (auto vendorPath = getVendorPath(url, regex)) {
-        return vendorPath;
-    } else {
-        return getIntegrationPath(url, "tiles/", regex);
-    }
-}
-
-mbgl::optional<std::string> localizeMapboxTilesetURL(const std::string& url) {
-    static const std::regex regex{"mapbox://"};
-    return getIntegrationPath(url, "tilesets/", regex);
-}
-
 void writeJSON(rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer, const mbgl::Value& value) {
     value.match([&writer](const mbgl::NullValue&) { writer.Null(); },
                 [&writer](bool b) { writer.Bool(b); },
@@ -248,7 +163,7 @@ JSONReply readJson(const mbgl::filesystem::path& jsonPath) {
         return { mbgl::formatJSONParseError(document) };
     }
 
-    return { std::move(document) };
+    return {std::move(document)};
 }
 
 std::string serializeJsonValue(const mbgl::JSValue& value) {
@@ -356,25 +271,6 @@ std::vector<std::string> readExpectedJSONEntries(const mbgl::filesystem::path& b
     return readExpectedEntries(regex, base);
 }
 
-namespace {
-
-std::vector<mbgl::filesystem::path> getTestExpectations(mbgl::filesystem::path testPath,
-                                                        const mbgl::filesystem::path& testsRootPath,
-                                                        std::vector<mbgl::filesystem::path> expectationsPaths) {
-    std::vector<mbgl::filesystem::path> expectations{std::move(testPath.remove_filename())};
-    const auto& defaultTestExpectationsPath = expectations.front().string();
-
-    const std::regex regex{testsRootPath.string()};
-    for (const auto& path : expectationsPaths) {
-        expectations.emplace_back(std::regex_replace(defaultTestExpectationsPath, regex, path.string()));
-        assert(!expectations.back().empty());
-    }
-
-    return expectations;
-}
-
-} // namespace
-
 ArgumentsTuple parseArguments(int argc, char** argv) {
     args::ArgumentParser argumentParser("Mapbox GL Test Runner");
 
@@ -384,7 +280,7 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
     args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order", {'s', "shuffle"});
     args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)", {"seed"});
     args::ValueFlag<std::string> testPathValue(argumentParser, "rootPath", "Test root rootPath", {'p', "rootPath"});
-    args::ValueFlag<std::regex> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"});
+    args::ValueFlag<std::string> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"});
     args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
 
     try {
@@ -419,73 +315,20 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
         exit(4);
     }
 
-    if (!ManifestParser::getInstance().parseManifest(testRootPath)) {
-        mbgl::Log::Error(
-            mbgl::Event::General, "Provided test rootPath '%s' does not exist.", rootPath.string().c_str());
+    const auto testNames = testNameValues ? args::get(testNameValues) : std::vector<std::string>{};
+    const auto testFilter = testFilterValue ? args::get(testFilterValue) : std::string{};
+    const auto shuffle = shuffleFlag ? args::get(shuffleFlag) : false;
+    const auto seed = seedValue ? args::get(seedValue) : 1u;
+    auto manifestData = ManifestParser::parseManifest(testRootPath, testNames, testFilter);
+    if (!manifestData) {
         exit(5);
     }
-
-    std::vector<mbgl::filesystem::path> paths;
-    const auto& baseTestPath = ManifestParser::getInstance().getBaseTestPath();
-    const auto& probeTestPath = ManifestParser::getInstance().getProbeTestPath();
-    if (!baseTestPath.empty()) {
-        for (const auto& id : args::get(testNameValues)) {
-            if (id == "tests") {
-                ManifestParser::getInstance().setTestPath(probeTestPath);
-                paths.emplace_back(probeTestPath / id);
-            } else {
-                paths.emplace_back(baseTestPath / id);
-            }
-        }
-        if (paths.empty()) {
-            paths.emplace_back(baseTestPath);
-        }
+    if (shuffle) {
+        manifestData->doShuffle(seed);
     }
 
-    const std::vector<mbgl::filesystem::path>& expectationsPaths = ManifestParser::getInstance().getExpectationsPaths();
-    // Recursively traverse through the test paths and collect test directories containing "style.json".
-    std::vector<TestPaths> testPaths;
-    testPaths.reserve(paths.size());
-    for (const auto& path : paths) {
-        if (!mbgl::filesystem::exists(path)) {
-            mbgl::Log::Warning(mbgl::Event::General, "Provided test folder '%s' does not exist.", path.string().c_str());
-            continue;
-        }
-        for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(path)) {
-            // Skip paths that fail regexp match.
-            if (testFilterValue && !std::regex_match(testPath.path().string(), args::get(testFilterValue))) {
-                continue;
-            }
-
-            if (testPath.path().filename() == "style.json") {
-                testPaths.emplace_back(testPath, getTestExpectations(testPath, path, expectationsPaths));
-            }
-        }
-    }
-
-    return ArgumentsTuple{recycleMapFlag ? args::get(recycleMapFlag) : false,
-                          shuffleFlag ? args::get(shuffleFlag) : false,
-                          seedValue ? args::get(seedValue) : 1u,
-                          std::move(testPaths)};
-}
-
-std::vector<std::pair<std::string, std::string>> parseIgnores() {
-    std::vector<std::pair<std::string, std::string>> ignores;
-    const std::vector<mbgl::filesystem::path>& ignoresPaths = ManifestParser::getInstance().getIgnoresPaths();
-
-    for (const auto& path : ignoresPaths) {
-        auto maybeIgnores = readJson(path);
-        if (!maybeIgnores.is<mbgl::JSDocument>()) {
-            continue;
-        }
-        for (const auto& property : maybeIgnores.get<mbgl::JSDocument>().GetObject()) {
-            const std::string ignore = {property.name.GetString(), property.name.GetStringLength()};
-            const std::string reason = {property.value.GetString(), property.value.GetStringLength()};
-            ignores.emplace_back(std::make_pair(ignore, reason));
-        }
-    }
-
-    return ignores;
+    return ArgumentsTuple{
+        recycleMapFlag ? args::get(recycleMapFlag) : false, shuffle, seed, testRootPath, std::move(manifestData)};
 }
 
 TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
@@ -577,7 +420,7 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
     return result;
 }
 
-TestMetadata parseTestMetadata(const TestPaths& paths) {
+TestMetadata parseTestMetadata(const TestPaths& paths, const Manifest& manifest) {
     TestMetadata metadata;
     metadata.paths = paths;
 
@@ -588,7 +431,7 @@ TestMetadata parseTestMetadata(const TestPaths& paths) {
     }
 
     metadata.document = std::move(maybeJson.get<mbgl::JSDocument>());
-    localizeStyleURLs(metadata.document, metadata.document);
+    manifest.localizeStyleURLs(metadata.document, metadata.document);
 
     if (!metadata.document.HasMember("metadata")) {
         mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s", paths.stylePath.c_str());
@@ -849,89 +692,3 @@ std::string createResultPage(const TestStatistics& stats, const std::vector<Test
 
     return resultsPage;
 }
-
-std::string localizeURL(const std::string& url) {
-    static const std::regex regex{"local://"};
-    if (auto vendorPath = getVendorPath(url, regex)) {
-        return *vendorPath;
-    } else {
-        return getIntegrationPath(url, "", regex).value_or(url);
-    }
-}
-
-void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) {
-    if (root.HasMember("urls") && root["urls"].IsArray()) {
-        for (auto& urlValue : root["urls"].GetArray()) {
-            const std::string path =
-                prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
-                                      .value_or(localizeLocalURL(urlValue.GetString()).value_or(urlValue.GetString())));
-            urlValue.Set<std::string>(path, document.GetAllocator());
-        }
-    }
-
-    if (root.HasMember("url")) {
-        static const std::string image("image");
-        static const std::string video("video");
-
-        mbgl::JSValue& urlValue = root["url"];
-        const std::string path =
-            prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
-                                  .value_or(localizeLocalURL(urlValue.GetString()).value_or(urlValue.GetString())));
-        urlValue.Set<std::string>(path, document.GetAllocator());
-
-        if (root["type"].GetString() != image && root["type"].GetString() != video) {
-            const auto tilesetPath = std::string(urlValue.GetString()).erase(0u, 7u); // remove "file://"
-            auto maybeTileset = readJson(tilesetPath);
-            if (maybeTileset.is<mbgl::JSDocument>()) {
-                const auto& tileset = maybeTileset.get<mbgl::JSDocument>();
-                assert(tileset.HasMember("tiles"));
-                root.AddMember("tiles", (mbgl::JSValue&)tileset["tiles"], document.GetAllocator());
-                root.RemoveMember("url");
-            }
-        }
-    }
-
-    if (root.HasMember("tiles")) {
-        mbgl::JSValue& tilesValue = root["tiles"];
-        assert(tilesValue.IsArray());
-        for (auto& tileValue : tilesValue.GetArray()) {
-            const std::string path = prependFileScheme(
-                localizeMapboxTilesURL(tileValue.GetString())
-                    .value_or(localizeLocalURL(tileValue.GetString())
-                                  .value_or(localizeHttpURL(tileValue.GetString()).value_or(tileValue.GetString()))));
-            tileValue.Set<std::string>(path, document.GetAllocator());
-        }
-    }
-
-    if (root.HasMember("data") && root["data"].IsString()) {
-        mbgl::JSValue& dataValue = root["data"];
-        const std::string path =
-            prependFileScheme(localizeLocalURL(dataValue.GetString()).value_or(dataValue.GetString()));
-        dataValue.Set<std::string>(path, document.GetAllocator());
-    }
-}
-
-void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) {
-    if (root.HasMember("sources")) {
-        mbgl::JSValue& sourcesValue = root["sources"];
-        for (auto& sourceProperty : sourcesValue.GetObject()) {
-            localizeSourceURLs(sourceProperty.value, document);
-        }
-    }
-
-    if (root.HasMember("glyphs")) {
-        mbgl::JSValue& glyphsValue = root["glyphs"];
-        const std::string path = prependFileScheme(
-            localizeMapboxFontsURL(glyphsValue.GetString())
-                .value_or(localizeLocalURL(glyphsValue.GetString(), true).value_or(glyphsValue.GetString())));
-        glyphsValue.Set<std::string>(path, document.GetAllocator());
-    }
-
-    if (root.HasMember("sprite")) {
-        mbgl::JSValue& spriteValue = root["sprite"];
-        const std::string path = prependFileScheme(
-            localizeMapboxSpriteURL(spriteValue.GetString())
-                .value_or(localizeLocalURL(spriteValue.GetString()).value_or(spriteValue.GetString())));
-        spriteValue.Set<std::string>(path, document.GetAllocator());
-    }
-}
diff --git a/render-test/parser.hpp b/render-test/parser.hpp
index 5b3e5c7d16..fefa032b76 100644
--- a/render-test/parser.hpp
+++ b/render-test/parser.hpp
@@ -2,6 +2,7 @@
 
 #include "metadata.hpp"
 
+#include <mbgl/util/optional.hpp>
 #include <mbgl/util/rapidjson.hpp>
 #include <mbgl/util/variant.hpp>
 
@@ -9,10 +10,12 @@
 #include <tuple>
 #include <vector>
 
+class Manifest;
+
 using ErrorMessage = std::string;
 using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
 
-using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::vector<TestPaths>>;
+using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, mbgl::optional<Manifest>>;
 
 JSONReply readJson(const mbgl::filesystem::path&);
 std::string serializeJsonValue(const mbgl::JSValue&);
@@ -24,16 +27,10 @@ std::vector<std::string> readExpectedJSONEntries(const mbgl::filesystem::path& b
 TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path);
 
 ArgumentsTuple parseArguments(int argc, char** argv);
-std::vector<std::pair<std::string, std::string>> parseIgnores();
 
-TestMetadata parseTestMetadata(const TestPaths& paths);
+TestMetadata parseTestMetadata(const TestPaths& paths, const Manifest& manifest);
 
 std::string createResultPage(const TestStatistics&, const std::vector<TestMetadata>&, bool shuffle, uint32_t seed);
 
-std::string localizeURL(const std::string& url);
-
 std::string toJSON(const mbgl::Value& value, unsigned indent, bool singleLine);
 std::string toJSON(const std::vector<mbgl::Feature>& features, unsigned indent, bool singleLine);
-
-void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document);
-void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document);
diff --git a/render-test/render_test.cpp b/render-test/render_test.cpp
index d23ec91b08..45f9871854 100644
--- a/render-test/render_test.cpp
+++ b/render-test/render_test.cpp
@@ -9,8 +9,6 @@
 #include "parser.hpp"
 #include "runner.hpp"
 
-#include <random>
-
 #define ANSI_COLOR_RED        "\x1b[31m"
 #define ANSI_COLOR_GREEN      "\x1b[32m"
 #define ANSI_COLOR_YELLOW     "\x1b[33m"
@@ -42,24 +40,23 @@ namespace mbgl {
 
 int runRenderTests(int argc, char** argv) {
     bool recycleMap;
+    mbgl::optional<Manifest> manifestData;
     bool shuffle;
     uint32_t seed;
-    std::vector<TestPaths> testPaths;
-
-    std::tie(recycleMap, shuffle, seed, testPaths) = parseArguments(argc, argv);
+    std::string manifestPath;
 
-    const auto ignores = parseIgnores();
+    std::tie(recycleMap, shuffle, seed, manifestPath, manifestData) = parseArguments(argc, argv);
+    assert(manifestData);
 
     if (shuffle) {
         printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed);
-
-        std::seed_seq sequence { seed };
-        std::mt19937 shuffler(sequence);
-        std::shuffle(testPaths.begin(), testPaths.end(), shuffler);
     }
+    const auto& manifest = manifestData.value();
+    const auto& ignores = manifest.getIgnores();
+    const auto& testPaths = manifest.getTestPaths();
 
     mbgl::util::RunLoop runLoop;
-    TestRunner runner{};
+    TestRunner runner(manifest);
 
     std::vector<TestMetadata> metadatas;
     metadatas.reserve(testPaths.size());
@@ -67,7 +64,7 @@ int runRenderTests(int argc, char** argv) {
     TestStatistics stats;
 
     for (auto& testPath : testPaths) {
-        TestMetadata metadata = parseTestMetadata(testPath);
+        TestMetadata metadata = parseTestMetadata(testPath, manifest);
 
         if (!recycleMap) {
             runner.reset();
@@ -77,7 +74,7 @@ int runRenderTests(int argc, char** argv) {
         std::string& status = metadata.status;
         std::string& color = metadata.color;
 
-        const std::string::size_type rootLength = ManifestParser::getInstance().getTestPath().length();
+        const std::string::size_type rootLength = manifest.getTestRootPath().length();
         id = testPath.defaultExpectations();
         id = id.substr(rootLength + 1, id.length() - rootLength - 2);
 
@@ -137,13 +134,12 @@ int runRenderTests(int argc, char** argv) {
 
         metadatas.push_back(std::move(metadata));
     }
-    const auto& testRootPath = ManifestParser::getInstance().getTestPath();
+    const auto& testRootPath = manifestPath;
     std::string resultsHTML = createResultPage(stats, metadatas, shuffle, seed);
     mbgl::util::write_file(testRootPath + "/index.html", resultsHTML);
 
-    const uint32_t count = stats.erroredTests + stats.failedTests +
-                           stats.ignoreFailedTests + stats.ignorePassedTests +
-                           stats.passedTests;
+    const uint32_t count =
+        stats.erroredTests + stats.failedTests + stats.ignoreFailedTests + stats.ignorePassedTests + stats.passedTests;
 
     if (stats.passedTests) {
         printf(ANSI_COLOR_GREEN "%u passed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.passedTests, 100.0 * stats.passedTests / count);
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index b41702b5d5..8def3bca96 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -102,6 +102,8 @@ std::string simpleDiff(const Value& result, const Value& expected) {
     return diff.str();
 }
 
+TestRunner::TestRunner(const Manifest& manifest_) : maps{}, manifest(manifest_) {}
+
 bool TestRunner::checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
                                        std::vector<mbgl::Feature>&& features,
                                        TestMetadata& metadata) {
@@ -486,8 +488,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         std::string imagePath = operationArray[2].GetString();
         imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end());
 
-        const mbgl::filesystem::path filePath =
-            mbgl::filesystem::path(ManifestParser::getInstance().getTestPath()) / imagePath;
+        const mbgl::filesystem::path filePath = mbgl::filesystem::path(manifest.getTestRootPath()) / imagePath;
 
         mbgl::optional<std::string> maybeImage = mbgl::util::readFile(filePath.string());
         if (!maybeImage) {
@@ -507,15 +508,15 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         // setStyle
         assert(operationArray.Size() >= 2u);
         if (operationArray[1].IsString()) {
-            std::string stylePath = localizeURL(operationArray[1].GetString());
+            std::string stylePath = manifest.localizeURL(operationArray[1].GetString());
             auto maybeStyle = readJson(stylePath);
             if (maybeStyle.is<mbgl::JSDocument>()) {
                 auto& style = maybeStyle.get<mbgl::JSDocument>();
-                localizeStyleURLs((mbgl::JSValue&)style, style);
+                manifest.localizeStyleURLs((mbgl::JSValue&)style, style);
                 map.getStyle().loadJSON(serializeJsonValue(style));
             }
         } else {
-            localizeStyleURLs(operationArray[1], metadata.document);
+            manifest.localizeStyleURLs(operationArray[1], metadata.document);
             map.getStyle().loadJSON(serializeJsonValue(operationArray[1]));
         }
     } else if (operationArray[0].GetString() == setCenterOp) {
@@ -616,7 +617,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         assert(operationArray[1].IsString());
         assert(operationArray[2].IsObject());
 
-        localizeSourceURLs(operationArray[2], metadata.document);
+        manifest.localizeSourceURLs(operationArray[2], metadata.document);
 
         mbgl::style::conversion::Error error;
         auto converted = mbgl::style::conversion::convert<std::unique_ptr<mbgl::style::Source>>(operationArray[2], error, operationArray[1].GetString());
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index 949c88af8c..bbcf536a88 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -11,10 +11,9 @@ struct TestMetadata;
 
 class TestRunner {
 public:
-    TestRunner() = default;
+    explicit TestRunner(const Manifest& manifest_);
     bool run(TestMetadata&);
     void reset();
-
 private:
     bool runOperations(const std::string& key, TestMetadata&);
     bool checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
@@ -31,4 +30,5 @@ private:
         mbgl::Map map;
     };
     std::unordered_map<std::string, std::unique_ptr<Impl>> maps;
+    const Manifest& manifest;
 };