[render-test] Parse full manifest file through CLI

14 files changed, 189 insertions, 172 deletions

diff --git a/next/platform/linux/linux.cmake b/next/platform/linux/linux.cmake
index 6ea1d4eb9f..9b821cdbea 100644
--- a/next/platform/linux/linux.cmake
+++ b/next/platform/linux/linux.cmake
@@ -144,9 +144,9 @@ add_test(
         render-tests
         --recycle-map
         --shuffle
-        --rootPath=${MBGL_ROOT}/render-test
+        --manifestPath=${MBGL_ROOT}/render-test/linux-manifest.json
         --seed=${MBGL_RENDER_TEST_SEED}
 )
 
-add_test(NAME mbgl-render-test-probes COMMAND mbgl-render-test-runner tests --rootPath=${MBGL_ROOT}/render-test)
-add_test(NAME mbgl-query-test COMMAND mbgl-render-test-runner query-tests --rootPath=${MBGL_ROOT}/render-test)
+add_test(NAME mbgl-render-test-probes COMMAND mbgl-render-test-runner tests --manifestPath=${MBGL_ROOT}/render-test/probe-manifest.json)
+add_test(NAME mbgl-query-test COMMAND mbgl-render-test-runner query-tests --manifestPath=${MBGL_ROOT}/render-test/linux-manifest.json)
diff --git a/next/platform/macos/macos.cmake b/next/platform/macos/macos.cmake
index 4838459c18..229b2cc9e3 100644
--- a/next/platform/macos/macos.cmake
+++ b/next/platform/macos/macos.cmake
@@ -208,10 +208,10 @@ add_test(
         render-tests
         --recycle-map
         --shuffle
-        --rootPath=${MBGL_ROOT}/render-test
+        --manifestPath=${MBGL_ROOT}/render-test/mac-manifest.json
         --seed=${MBGL_RENDER_TEST_SEED}
 )
 
-add_test(NAME mbgl-render-test-probes COMMAND mbgl-render-test-runner tests --rootPath=${MBGL_ROOT}/render-test)
+add_test(NAME mbgl-render-test-probes COMMAND mbgl-render-test-runner tests --manifestPath=${MBGL_ROOT}/render-test/probe-manifest.json)
 
-add_test(NAME mbgl-query-test COMMAND mbgl-render-test-runner query-tests --rootPath=${MBGL_ROOT}/render-test)
+add_test(NAME mbgl-query-test COMMAND mbgl-render-test-runner query-tests --manifestPath=${MBGL_ROOT}/render-test/mac-manifest.json)
diff --git a/platform/android/src/test/render_test_runner.cpp b/platform/android/src/test/render_test_runner.cpp
index 3f85140128..d4554aa9de 100644
--- a/platform/android/src/test/render_test_runner.cpp
+++ b/platform/android/src/test/render_test_runner.cpp
@@ -46,7 +46,7 @@ void android_main(struct android_app* app) {
     JNIEnv* env;
     app->activity->vm->AttachCurrentThread(&env, NULL);
 
-    std::vector<std::string> arguments = {"mbgl-render-test-runner", "-p", "/sdcard/render-test"};
+    std::vector<std::string> arguments = {"mbgl-render-test-runner", "-p", "/sdcard/render-test/android-manifest.json"};
     std::vector<char*> argv;
     for (const auto& arg : arguments) {
         argv.push_back((char*)arg.data());
diff --git a/render-test/android-manifest.json b/render-test/android-manifest.json
index dcd013eee5..56223d4753 100644
--- a/render-test/android-manifest.json
+++ b/render-test/android-manifest.json
@@ -1,6 +1,5 @@
 {
     "base_test_path":"mapbox-gl-js/test/integration",
-    "probe_test_path":".",
     "expectation_paths":["render-test/expected/render-tests"],
     "ignore_paths":["platform/node/test/ignores.json", "render-test/linux-ignores.json", "render-test/tests/should-fail.json"],
     "vendor_path":"vendor",
diff --git a/render-test/linux-manifest.json b/render-test/linux-manifest.json
index e0b8e101d8..32a5afdbdb 100644
--- a/render-test/linux-manifest.json
+++ b/render-test/linux-manifest.json
@@ -1,6 +1,5 @@
 {
     "base_test_path":"../mapbox-gl-js/test/integration",
-    "probe_test_path":".",
     "expectation_paths":["expected/render-tests"],
     "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json", "../render-test/tests/should-fail.json"],
     "vendor_path":"../vendor",
diff --git a/render-test/mac-manifest.json b/render-test/mac-manifest.json
index b53f841506..224df81298 100644
--- a/render-test/mac-manifest.json
+++ b/render-test/mac-manifest.json
@@ -1,6 +1,5 @@
 {
     "base_test_path":"../mapbox-gl-js/test/integration",
-    "probe_test_path":".",
     "expectation_paths":["expected/render-tests", "tests/mac"],
     "ignore_paths":["../platform/node/test/ignores.json", "../render-test/mac-ignores.json", "../render-test/tests/should-fail.json"],
     "vendor_path":"../vendor",
diff --git a/render-test/manifest_parser.cpp b/render-test/manifest_parser.cpp
index f7398a9856..f1884634e5 100644
--- a/render-test/manifest_parser.cpp
+++ b/render-test/manifest_parser.cpp
@@ -4,6 +4,7 @@
 
 #include <mbgl/util/logging.hpp>
 
+#include <algorithm>
 #include <random>
 
 namespace {
@@ -21,6 +22,9 @@ std::string prependFileScheme(const std::string& url) {
 }
 } // namespace
 
+Manifest::Manifest() = default;
+Manifest::~Manifest() = default;
+
 const std::vector<TestPaths>& Manifest::getTestPaths() const {
     return testPaths;
 }
@@ -30,6 +34,9 @@ const std::vector<std::pair<std::string, std::string>>& Manifest::getIgnores() c
 const std::string& Manifest::getTestRootPath() const {
     return testRootPath;
 }
+const std::string& Manifest::getManifestPath() const {
+    return manifestPath;
+}
 
 void Manifest::doShuffle(uint32_t seed) {
     std::seed_seq sequence{seed};
@@ -41,9 +48,8 @@ std::string Manifest::localizeURL(const std::string& url) const {
     static const std::regex regex{"local://"};
     if (auto vendorPath = getVendorPath(url, regex)) {
         return *vendorPath;
-    } else {
-        return getIntegrationPath(url, "", regex).value_or(url);
     }
+    return getIntegrationPath(url, "", regex).value_or(url);
 }
 
 void Manifest::localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const {
@@ -127,18 +133,16 @@ mbgl::optional<std::string> Manifest::localizeLocalURL(const std::string& url, b
     static const std::regex regex{"local://"};
     if (auto vendorPath = getVendorPath(url, regex, glyphsPath)) {
         return vendorPath;
-    } else {
-        return getIntegrationPath(url, "", regex, glyphsPath);
     }
+    return getIntegrationPath(url, "", regex, glyphsPath);
 }
 
 mbgl::optional<std::string> Manifest::localizeHttpURL(const std::string& url) const {
     static const std::regex regex{"http://localhost:2900"};
     if (auto vendorPath = getVendorPath(url, regex)) {
         return vendorPath;
-    } else {
-        return getIntegrationPath(url, "", regex);
     }
+    return getIntegrationPath(url, "", regex);
 }
 
 mbgl::optional<std::string> Manifest::localizeMapboxSpriteURL(const std::string& url) const {
@@ -155,9 +159,8 @@ mbgl::optional<std::string> Manifest::localizeMapboxTilesURL(const std::string&
     static const std::regex regex{"mapbox://"};
     if (auto vendorPath = getVendorPath(url, regex)) {
         return vendorPath;
-    } else {
-        return getIntegrationPath(url, "tiles/", regex);
     }
+    return getIntegrationPath(url, "tiles/", regex);
 }
 
 mbgl::optional<std::string> Manifest::localizeMapboxTilesetURL(const std::string& url) const {
@@ -177,7 +180,7 @@ mbgl::optional<std::string> Manifest::getVendorPath(const std::string& url,
         return removeURLArguments(file.string());
     }
 
-    return {};
+    return mbgl::nullopt;
 }
 
 mbgl::optional<std::string> Manifest::getIntegrationPath(const std::string& url,
@@ -193,7 +196,7 @@ mbgl::optional<std::string> Manifest::getIntegrationPath(const std::string& url,
         return removeURLArguments(file.string());
     }
 
-    return {};
+    return mbgl::nullopt;
 }
 
 namespace {
@@ -229,8 +232,8 @@ std::vector<mbgl::filesystem::path> getTestExpectations(mbgl::filesystem::path t
     return expectations;
 }
 
-mbgl::filesystem::path getValidPath(const std::string& rootPath, const std::string& path) {
-    const static mbgl::filesystem::path BasePath{rootPath};
+mbgl::filesystem::path getValidPath(const std::string& manifestPath, const std::string& path) {
+    const static mbgl::filesystem::path BasePath{manifestPath};
     mbgl::filesystem::path result{path};
     if (result.is_relative()) {
         result = BasePath / result;
@@ -244,26 +247,17 @@ mbgl::filesystem::path getValidPath(const std::string& rootPath, const std::stri
 
 } // namespace
 
-mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& rootPath,
+mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manifestPath,
                                                        const std::vector<std::string>& testNames,
                                                        const std::string& testFilter) {
-    static Manifest manifest;
-    auto filePath = mbgl::filesystem::path(rootPath);
-#ifdef __APPLE__
-    filePath /= "mac-manifest.json";
-#elif __ANDROID__
-    filePath /= "android-manifest.json";
-#else
-    filePath /= "linux-manifest.json";
-#endif
-    if (!mbgl::filesystem::exists(filePath)) {
-        mbgl::Log::Error(mbgl::Event::General, "Provided manifest file: %s is not valid", filePath.c_str());
-        return {};
-    }
+    Manifest manifest;
+    const auto filePath = mbgl::filesystem::path(manifestPath);
+    manifest.manifestPath = manifestPath.substr(0, manifestPath.find(filePath.filename()));
+
     auto contents = readJson(filePath);
     if (!contents.is<mbgl::JSDocument>()) {
         mbgl::Log::Error(mbgl::Event::General, "Provided manifest file: %s is not a valid json", filePath.c_str());
-        return {};
+        return mbgl::nullopt;
     }
 
     auto document = std::move(contents.get<mbgl::JSDocument>());
@@ -272,12 +266,11 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& rootPa
         if (!assetPathValue.IsString()) {
             mbgl::Log::Warning(
                 mbgl::Event::General, "Invalid assetPath is provoided inside the manifest file: %s", filePath.c_str());
-            return {};
+            return mbgl::nullopt;
         }
-
-        manifest.assetPath = getValidPath(rootPath, assetPathValue.GetString()).string();
-        if (manifest.assetPath.back() != '/') {
-            manifest.assetPath.push_back('/');
+        manifest.assetPath = (getValidPath(manifest.manifestPath, assetPathValue.GetString()) / "").string();
+        if (manifest.assetPath.empty()) {
+            return mbgl::nullopt;
         }
     }
     if (document.HasMember("vendor_path")) {
@@ -285,13 +278,11 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& rootPa
         if (!vendorPathValue.IsString()) {
             mbgl::Log::Warning(
                 mbgl::Event::General, "Invalid vendorPath is provoided inside the manifest file: %s", filePath.c_str());
-            return {};
+            return mbgl::nullopt;
         }
-        auto path = std::string(vendorPathValue.GetString());
-
-        manifest.vendorPath = getValidPath(rootPath, vendorPathValue.GetString()).string();
-        if (manifest.vendorPath.back() != '/') {
-            manifest.vendorPath.push_back('/');
+        manifest.vendorPath = (getValidPath(manifest.manifestPath, vendorPathValue.GetString()) / "").string();
+        if (manifest.vendorPath.empty()) {
+            return mbgl::nullopt;
         }
     }
     mbgl::filesystem::path baseTestPath;
@@ -300,44 +291,47 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& rootPa
         if (!testPathValue.IsString()) {
             mbgl::Log::Warning(
                 mbgl::Event::General, "Invalid testPath is provoided inside the manifest file: %s", filePath.c_str());
-            return {};
+            return mbgl::nullopt;
         }
-        auto path = getValidPath(rootPath, testPathValue.GetString()).string();
-        if (path.back() == '/') {
-            path.pop_back();
+        baseTestPath = getValidPath(manifest.manifestPath, testPathValue.GetString());
+        if (baseTestPath.empty()) {
+            return mbgl::nullopt;
         }
-        baseTestPath = mbgl::filesystem::path(path);
     }
     mbgl::filesystem::path probeTestPath;
+    bool enbaleProbeTest{false};
     if (document.HasMember("probe_test_path")) {
         const auto& testPathValue = document["probe_test_path"];
         if (!testPathValue.IsString()) {
             mbgl::Log::Warning(
                 mbgl::Event::General, "Invalid testPath is provoided inside the manifest file: %s", filePath.c_str());
-            return {};
+            return mbgl::nullopt;
         }
-        auto path = getValidPath(rootPath, testPathValue.GetString()).string();
-        if (path.back() == '/') {
-            path.pop_back();
+        probeTestPath = getValidPath(manifest.manifestPath, testPathValue.GetString());
+        if (probeTestPath.empty()) {
+            return mbgl::nullopt;
         }
-        probeTestPath = mbgl::filesystem::path(path);
+        enbaleProbeTest = true;
     }
     std::vector<mbgl::filesystem::path> expectationPaths{};
     if (document.HasMember("expectation_paths")) {
         const auto& expectationPathValue = document["expectation_paths"];
         if (!expectationPathValue.IsArray()) {
             mbgl::Log::Warning(mbgl::Event::General,
-                               "Invalid expectationPath is provoided inside the manifest file: %s",
+                               "Provided expectation_paths inside the manifest file: %s is not a valid array",
                                filePath.c_str());
-            return {};
+            return mbgl::nullopt;
         }
         for (const auto& value : expectationPathValue.GetArray()) {
             if (!value.IsString()) {
-                return {};
+                mbgl::Log::Warning(mbgl::Event::General,
+                                   "Invalid expectation path item is provoided inside the manifest file: %s",
+                                   filePath.c_str());
+                return mbgl::nullopt;
             }
-            expectationPaths.emplace_back(getValidPath(rootPath, value.GetString()));
+            expectationPaths.emplace_back(getValidPath(manifest.manifestPath, value.GetString()));
             if (expectationPaths.back().empty()) {
-                return {};
+                return mbgl::nullopt;
             }
         }
     }
@@ -345,36 +339,43 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& rootPa
     if (document.HasMember("ignore_paths")) {
         const auto& ignorePathValue = document["ignore_paths"];
         if (!ignorePathValue.IsArray()) {
-            mbgl::Log::Warning(
-                mbgl::Event::General, "Invalid ignorePath is provoided inside the manifest file: %s", filePath.c_str());
-            return {};
+            mbgl::Log::Warning(mbgl::Event::General,
+                               "Provided ignore_paths inside the manifest file: %s is not a valid array",
+                               filePath.c_str());
+            return mbgl::nullopt;
         }
         for (const auto& value : ignorePathValue.GetArray()) {
             if (!value.IsString()) {
-                return {};
+                mbgl::Log::Warning(mbgl::Event::General,
+                                   "Invalid ignore path item is provoided inside the manifest file: %s",
+                                   filePath.c_str());
+                return mbgl::nullopt;
             }
-            ignorePaths.emplace_back(getValidPath(rootPath, value.GetString()));
+            ignorePaths.emplace_back(getValidPath(manifest.manifestPath, value.GetString()));
             if (ignorePaths.back().empty()) {
-                return {};
+                return mbgl::nullopt;
             }
         }
         manifest.ignores = parseIgnores(ignorePaths);
     }
-    bool enbaleProbeTest{false};
+
+    manifest.testRootPath = enbaleProbeTest ? probeTestPath.string() : baseTestPath.string();
+    if (manifest.testRootPath.back() == '/') {
+        manifest.testRootPath.pop_back();
+    }
+    if (manifest.manifestPath.back() == '/') {
+        manifest.manifestPath.pop_back();
+    }
+
     std::vector<mbgl::filesystem::path> paths;
     for (const auto& id : testNames) {
-        if (id == "tests") {
-            paths.emplace_back(probeTestPath / id);
-            enbaleProbeTest = true;
-        } else {
-            paths.emplace_back(baseTestPath / id);
-        }
+        paths.emplace_back(manifest.testRootPath + "/" + id);
     }
     if (paths.empty()) {
-        paths.emplace_back(baseTestPath);
+        paths.emplace_back(manifest.testRootPath);
     }
 
-    // Recursively traverse through the test paths and collect test directories containing "style.son".
+    // Recursively traverse through the test paths and collect test directories containing "style.json".
     auto& testPaths = manifest.testPaths;
     testPaths.reserve(paths.size());
     for (const auto& path : paths) {
@@ -395,9 +396,5 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& rootPa
         }
     }
 
-    manifest.testRootPath = enbaleProbeTest ? probeTestPath.string() : baseTestPath.string();
-    if (manifest.testRootPath.back() == '/') {
-        manifest.testRootPath.pop_back();
-    }
     return mbgl::optional<Manifest>(manifest);
 }
diff --git a/render-test/manifest_parser.hpp b/render-test/manifest_parser.hpp
index 7c96a6e6c4..bc5adf1091 100644
--- a/render-test/manifest_parser.hpp
+++ b/render-test/manifest_parser.hpp
@@ -10,11 +10,14 @@
 #include <utility>
 #include <vector>
 
-struct Manifest {
+class Manifest {
 public:
+    Manifest();
+    ~Manifest();
     const std::vector<std::pair<std::string, std::string>>& getIgnores() const;
     const std::vector<TestPaths>& getTestPaths() const;
     const std::string& getTestRootPath() const;
+    const std::string& getManifestPath() const;
     void doShuffle(uint32_t seed);
 
     std::string localizeURL(const std::string& url) const;
@@ -36,16 +39,17 @@ private:
                                                    const std::string& parent,
                                                    const std::regex& regex,
                                                    bool glyphsPath = false) const;
-    std::string testRootPath{};
-    std::string vendorPath{};
-    std::string assetPath{};
-    std::vector<std::pair<std::string, std::string>> ignores{};
-    std::vector<TestPaths> testPaths{};
+    std::string manifestPath;
+    std::string testRootPath;
+    std::string vendorPath;
+    std::string assetPath;
+    std::vector<std::pair<std::string, std::string>> ignores;
+    std::vector<TestPaths> testPaths;
 };
 
 class ManifestParser {
 public:
-    static mbgl::optional<Manifest> parseManifest(const std::string& rootPath_,
+    static mbgl::optional<Manifest> parseManifest(const std::string& manifestPath,
                                                   const std::vector<std::string>& testNames,
                                                   const std::string& testFilter);
 };
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index b3ace13d5c..5a91fc7a58 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -3,8 +3,6 @@
 #include <mbgl/util/rapidjson.hpp>
 #include <mbgl/util/string.hpp>
 
-#include <args.hxx>
-
 #include <rapidjson/prettywriter.h>
 #include <rapidjson/stringbuffer.h>
 #include <rapidjson/writer.h>
@@ -19,7 +17,6 @@
 #include <boost/archive/iterators/ostream_iterator.hpp>
 
 #include "filesystem.hpp"
-#include "manifest_parser.hpp"
 #include "metadata.hpp"
 #include "parser.hpp"
 #include "runner.hpp"
@@ -271,66 +268,6 @@ std::vector<std::string> readExpectedJSONEntries(const mbgl::filesystem::path& b
     return readExpectedEntries(regex, base);
 }
 
-ArgumentsTuple parseArguments(int argc, char** argv) {
-    args::ArgumentParser argumentParser("Mapbox GL Test Runner");
-
-    args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", { 'h', "help" });
-
-    args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", {'r', "recycle-map"});
-    args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order", {'s', "shuffle"});
-    args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)", {"seed"});
-    args::ValueFlag<std::string> testPathValue(argumentParser, "rootPath", "Test root rootPath", {'p', "rootPath"});
-    args::ValueFlag<std::string> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"});
-    args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
-
-    try {
-        argumentParser.ParseCLI(argc, argv);
-    } catch (const args::Help&) {
-        std::ostringstream stream;
-        stream << argumentParser;
-        mbgl::Log::Info(mbgl::Event::General, stream.str());
-        exit(0);
-    } catch (const args::ParseError& e) {
-        std::ostringstream stream;
-        stream << argumentParser;
-        mbgl::Log::Info(mbgl::Event::General, stream.str());
-        mbgl::Log::Error(mbgl::Event::General, e.what());
-        exit(1);
-    } catch (const args::ValidationError& e) {
-        std::ostringstream stream;
-        stream << argumentParser;
-        mbgl::Log::Info(mbgl::Event::General, stream.str());
-        mbgl::Log::Error(mbgl::Event::General, e.what());
-        exit(2);
-    } catch (const std::regex_error& e) {
-        mbgl::Log::Error(mbgl::Event::General, "Invalid filter regular expression: %s", e.what());
-        exit(3);
-    }
-
-    const auto testRootPath = testPathValue ? args::get(testPathValue) : std::string{TEST_RUNNER_ROOT_PATH};
-    mbgl::filesystem::path rootPath{testRootPath};
-    if (!mbgl::filesystem::exists(rootPath)) {
-        mbgl::Log::Error(
-            mbgl::Event::General, "Provided test rootPath '%s' does not exist.", rootPath.string().c_str());
-        exit(4);
-    }
-
-    const auto testNames = testNameValues ? args::get(testNameValues) : std::vector<std::string>{};
-    const auto testFilter = testFilterValue ? args::get(testFilterValue) : std::string{};
-    const auto shuffle = shuffleFlag ? args::get(shuffleFlag) : false;
-    const auto seed = seedValue ? args::get(seedValue) : 1u;
-    auto manifestData = ManifestParser::parseManifest(testRootPath, testNames, testFilter);
-    if (!manifestData) {
-        exit(5);
-    }
-    if (shuffle) {
-        manifestData->doShuffle(seed);
-    }
-
-    return ArgumentsTuple{
-        recycleMapFlag ? args::get(recycleMapFlag) : false, shuffle, seed, testRootPath, std::move(manifestData)};
-}
-
 TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
     TestMetrics result;
 
@@ -576,7 +513,8 @@ std::string encodeBase64(const std::string& data) {
 }
 
 std::string createResultItem(const TestMetadata& metadata, bool hasFailedTests) {
-    const bool shouldHide = (hasFailedTests && metadata.status == "passed") || (metadata.status.find("ignored") != std::string::npos);
+    const bool shouldHide =
+        (hasFailedTests && metadata.status == "passed") || (metadata.status.find("ignored") != std::string::npos);
 
     std::string html;
     html.append("<div class=\"test " + metadata.status + (shouldHide ? " hide" : "") + "\">\n");
diff --git a/render-test/parser.hpp b/render-test/parser.hpp
index fefa032b76..3d79ac668a 100644
--- a/render-test/parser.hpp
+++ b/render-test/parser.hpp
@@ -2,7 +2,6 @@
 
 #include "metadata.hpp"
 
-#include <mbgl/util/optional.hpp>
 #include <mbgl/util/rapidjson.hpp>
 #include <mbgl/util/variant.hpp>
 
@@ -15,8 +14,6 @@ class Manifest;
 using ErrorMessage = std::string;
 using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
 
-using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, mbgl::optional<Manifest>>;
-
 JSONReply readJson(const mbgl::filesystem::path&);
 std::string serializeJsonValue(const mbgl::JSValue&);
 std::string serializeMetrics(const TestMetrics&);
@@ -26,8 +23,6 @@ std::vector<std::string> readExpectedJSONEntries(const mbgl::filesystem::path& b
 
 TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path);
 
-ArgumentsTuple parseArguments(int argc, char** argv);
-
 TestMetadata parseTestMetadata(const TestPaths& paths, const Manifest& manifest);
 
 std::string createResultPage(const TestStatistics&, const std::vector<TestMetadata>&, bool shuffle, uint32_t seed);
diff --git a/render-test/probe-manifest.json b/render-test/probe-manifest.json
new file mode 100644
index 0000000000..f3cc56d0a8
--- /dev/null
+++ b/render-test/probe-manifest.json
@@ -0,0 +1,7 @@
+{
+    "probe_test_path":".",
+    "expectation_paths":["expected/render-tests", "tests/mac"],
+    "ignore_paths":["../render-test/mac-ignores.json", "../render-test/tests/should-fail.json"],
+    "vendor_path":"../vendor",
+    "asset_path": "../mapbox-gl-js/test/integration"
+}
\ No newline at end of file
diff --git a/render-test/render_test.cpp b/render-test/render_test.cpp
index 45f9871854..85c716dabb 100644
--- a/render-test/render_test.cpp
+++ b/render-test/render_test.cpp
@@ -2,8 +2,11 @@
 
 #include <mbgl/render_test.hpp>
 #include <mbgl/util/io.hpp>
+#include <mbgl/util/logging.hpp>
 #include <mbgl/util/run_loop.hpp>
 
+#include <args.hxx>
+
 #include "manifest_parser.hpp"
 #include "metadata.hpp"
 #include "parser.hpp"
@@ -36,28 +39,90 @@ void operator delete(void* ptr, size_t) noexcept {
 }
 #endif
 
+namespace {
+using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>, std::string>;
+ArgumentsTuple parseArguments(int argc, char** argv) {
+    args::ArgumentParser argumentParser("Mapbox GL Test Runner");
+
+    args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", {'h', "help"});
+
+    args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", {'r', "recycle-map"});
+    args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order", {'s', "shuffle"});
+    args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)", {"seed"});
+    args::ValueFlag<std::string> testPathValue(
+        argumentParser, "manifestPath", "Test manifest file path", {'p', "manifestPath"});
+    args::ValueFlag<std::string> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"});
+    args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
+
+    try {
+        argumentParser.ParseCLI(argc, argv);
+    } catch (const args::Help&) {
+        std::ostringstream stream;
+        stream << argumentParser;
+        mbgl::Log::Info(mbgl::Event::General, stream.str());
+        exit(0);
+    } catch (const args::ParseError& e) {
+        std::ostringstream stream;
+        stream << argumentParser;
+        mbgl::Log::Info(mbgl::Event::General, stream.str());
+        mbgl::Log::Error(mbgl::Event::General, e.what());
+        exit(1);
+    } catch (const args::ValidationError& e) {
+        std::ostringstream stream;
+        stream << argumentParser;
+        mbgl::Log::Info(mbgl::Event::General, stream.str());
+        mbgl::Log::Error(mbgl::Event::General, e.what());
+        exit(2);
+    } catch (const std::regex_error& e) {
+        mbgl::Log::Error(mbgl::Event::General, "Invalid filter regular expression: %s", e.what());
+        exit(3);
+    }
+
+    mbgl::filesystem::path manifestPath{testPathValue ? args::get(testPathValue) : std::string{TEST_RUNNER_ROOT_PATH}};
+    if (!mbgl::filesystem::exists(manifestPath) || !manifestPath.has_filename()) {
+        mbgl::Log::Error(mbgl::Event::General,
+                         "Provided test manifest file path '%s' does not exist",
+                         manifestPath.string().c_str());
+        exit(4);
+    }
+
+    auto testNames = testNameValues ? args::get(testNameValues) : std::vector<std::string>{};
+    auto testFilter = testFilterValue ? args::get(testFilterValue) : std::string{};
+    const auto shuffle = shuffleFlag ? args::get(shuffleFlag) : false;
+    const auto seed = seedValue ? args::get(seedValue) : 1u;
+    return ArgumentsTuple{recycleMapFlag ? args::get(recycleMapFlag) : false,
+                          shuffle,
+                          seed,
+                          manifestPath.string(),
+                          std::move(testNames),
+                          std::move(testFilter)};
+}
+} // namespace
 namespace mbgl {
 
 int runRenderTests(int argc, char** argv) {
     bool recycleMap;
-    mbgl::optional<Manifest> manifestData;
     bool shuffle;
     uint32_t seed;
     std::string manifestPath;
+    std::vector<std::string> testNames;
+    std::string testFilter;
 
-    std::tie(recycleMap, shuffle, seed, manifestPath, manifestData) = parseArguments(argc, argv);
-    assert(manifestData);
-
+    std::tie(recycleMap, shuffle, seed, manifestPath, testNames, testFilter) = parseArguments(argc, argv);
+    auto manifestData = ManifestParser::parseManifest(manifestPath, testNames, testFilter);
+    if (!manifestData) {
+        exit(5);
+    }
+    mbgl::util::RunLoop runLoop;
+    TestRunner runner(std::move(*manifestData));
     if (shuffle) {
         printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed);
+        runner.doShuffle(seed);
     }
-    const auto& manifest = manifestData.value();
+
+    const auto& manifest = runner.getManifest();
     const auto& ignores = manifest.getIgnores();
     const auto& testPaths = manifest.getTestPaths();
-
-    mbgl::util::RunLoop runLoop;
-    TestRunner runner(manifest);
-
     std::vector<TestMetadata> metadatas;
     metadatas.reserve(testPaths.size());
 
@@ -134,7 +199,7 @@ int runRenderTests(int argc, char** argv) {
 
         metadatas.push_back(std::move(metadata));
     }
-    const auto& testRootPath = manifestPath;
+    const auto& testRootPath = manifest.getManifestPath();
     std::string resultsHTML = createResultPage(stats, metadatas, shuffle, seed);
     mbgl::util::write_file(testRootPath + "/index.html", resultsHTML);
 
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 8def3bca96..e882e394be 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -23,7 +23,6 @@
 #include <../expression-test/test_runner_common.hpp>
 #include "allocation_index.hpp"
 #include "file_source.hpp"
-#include "manifest_parser.hpp"
 #include "metadata.hpp"
 #include "parser.hpp"
 #include "runner.hpp"
@@ -102,7 +101,15 @@ std::string simpleDiff(const Value& result, const Value& expected) {
     return diff.str();
 }
 
-TestRunner::TestRunner(const Manifest& manifest_) : maps{}, manifest(manifest_) {}
+TestRunner::TestRunner(Manifest manifest_) : manifest(std::move(manifest_)) {}
+
+const Manifest& TestRunner::getManifest() const {
+    return manifest;
+}
+
+void TestRunner::doShuffle(uint32_t seed) {
+    manifest.doShuffle(seed);
+}
 
 bool TestRunner::checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
                                        std::vector<mbgl::Feature>&& features,
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index bbcf536a88..bc97f8300b 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -3,6 +3,8 @@
 #include <mbgl/gfx/headless_frontend.hpp>
 #include <mbgl/map/map.hpp>
 
+#include "manifest_parser.hpp"
+
 #include <memory>
 #include <string>
 
@@ -11,9 +13,14 @@ struct TestMetadata;
 
 class TestRunner {
 public:
-    explicit TestRunner(const Manifest& manifest_);
+    explicit TestRunner(Manifest);
     bool run(TestMetadata&);
     void reset();
+
+    // Manifest
+    const Manifest& getManifest() const;
+    void doShuffle(uint32_t seed);
+
 private:
     bool runOperations(const std::string& key, TestMetadata&);
     bool checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
@@ -30,5 +37,5 @@ private:
         mbgl::Map map;
     };
     std::unordered_map<std::string, std::unique_ptr<Impl>> maps;
-    const Manifest& manifest;
+    Manifest manifest;
 };