summaryrefslogtreecommitdiff
path: root/render-test
diff options
context:
space:
mode:
authorzmiao <miao.zhao@mapbox.com>2019-10-29 20:59:29 +0200
committerGitHub <noreply@github.com>2019-10-29 20:59:29 +0200
commitc8576e089f12438a2384a4fe822af650e94ace74 (patch)
treead69be13bc00c2dbece7cceedd5007b7532fe173 /render-test
parent84f2ff1084ccaec68c7a26243367ae657f6ebe60 (diff)
downloadqtlocation-mapboxgl-c8576e089f12438a2384a4fe822af650e94ace74.tar.gz
[render-test] Add manifest parser for render-test-runner (#15861)
* [render-test] Add manifest parser for render-test-runner * [render-test] Refactory manifest parser * [render-test] Parse full manifest file through CLI * [render-test] Add linux probe manifest
Diffstat (limited to 'render-test')
-rw-r--r--render-test/android-manifest.json7
-rwxr-xr-xrender-test/android/render_test_setup.sh10
-rw-r--r--render-test/linux-manifest.json7
-rw-r--r--render-test/linux-probe-manifest.json7
-rw-r--r--render-test/mac-manifest.json7
-rw-r--r--render-test/mac-probe-manifest.json7
-rw-r--r--render-test/manifest_parser.cpp400
-rw-r--r--render-test/manifest_parser.hpp55
-rw-r--r--render-test/parser.cpp398
-rw-r--r--render-test/parser.hpp20
-rw-r--r--render-test/render_test.cpp112
-rw-r--r--render-test/runner.cpp20
-rw-r--r--render-test/runner.hpp11
13 files changed, 621 insertions, 440 deletions
diff --git a/render-test/android-manifest.json b/render-test/android-manifest.json
new file mode 100644
index 0000000000..56223d4753
--- /dev/null
+++ b/render-test/android-manifest.json
@@ -0,0 +1,7 @@
+{
+ "base_test_path":"mapbox-gl-js/test/integration",
+ "expectation_paths":["render-test/expected/render-tests"],
+ "ignore_paths":["platform/node/test/ignores.json", "render-test/linux-ignores.json", "render-test/tests/should-fail.json"],
+ "vendor_path":"vendor",
+ "asset_path": "mapbox-gl-js/test/integration"
+} \ No newline at end of file
diff --git a/render-test/android/render_test_setup.sh b/render-test/android/render_test_setup.sh
index e34d71df3b..1dea44399e 100755
--- a/render-test/android/render_test_setup.sh
+++ b/render-test/android/render_test_setup.sh
@@ -3,8 +3,8 @@
adb shell rm -rf /sdcard/render-test
adb shell mkdir /sdcard/render-test
adb shell mkdir /sdcard/render-test/vendor
-adb shell mkdir /sdcard/render-test/expected
adb shell mkdir /sdcard/render-test/ignores
+adb shell mkdir /sdcard/render-test/render-test/tests
# push test sources
adb push ../../mapbox-gl-js/test/integration/render-tests /sdcard/render-test/mapbox-gl-js/test/integration/render-tests
@@ -25,7 +25,7 @@ adb push sprites /sdcard/render-test/mapbox-gl-js/test/integration/sprites
rm -rf sprites
# push extra expectations
-adb push ../../render-test/expected /sdcard/render-test/render-test/expected
+adb push ../../render-test/expected/render-tests /sdcard/render-test/render-test/expected/render-tests
# push default ignore lists
adb shell mkdir /sdcard/render-test/platform
@@ -33,6 +33,10 @@ adb shell mkdir /sdcard/render-test/platform/node
adb shell mkdir /sdcard/render-test/platform/node/test
adb push ../../platform/node/test/ignores.json /sdcard/render-test/platform/node/test
adb shell mkdir /sdcard/render-test/render-test
-adb push ../../render-test/linux-ignores.json /sdcard/render-test/render-test
+adb push ../linux-ignores.json /sdcard/render-test/render-test
+adb push ../tests/should-fail.json /sdcard/render-test/render-test/tests
+
+# push manifest
+adb push ../android-manifest.json /sdcard/render-test
adb shell ls /sdcard/render-test/ \ No newline at end of file
diff --git a/render-test/linux-manifest.json b/render-test/linux-manifest.json
new file mode 100644
index 0000000000..32a5afdbdb
--- /dev/null
+++ b/render-test/linux-manifest.json
@@ -0,0 +1,7 @@
+{
+ "base_test_path":"../mapbox-gl-js/test/integration",
+ "expectation_paths":["expected/render-tests"],
+ "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json", "../render-test/tests/should-fail.json"],
+ "vendor_path":"../vendor",
+ "asset_path": "../mapbox-gl-js/test/integration"
+} \ No newline at end of file
diff --git a/render-test/linux-probe-manifest.json b/render-test/linux-probe-manifest.json
new file mode 100644
index 0000000000..0b2ca88988
--- /dev/null
+++ b/render-test/linux-probe-manifest.json
@@ -0,0 +1,7 @@
+{
+ "probe_test_path":".",
+ "expectation_paths":["expected/render-tests"],
+ "ignore_paths":["../render-test/linux-ignores.json", "../render-test/tests/should-fail.json"],
+ "vendor_path":"../vendor",
+ "asset_path": "../mapbox-gl-js/test/integration"
+} \ No newline at end of file
diff --git a/render-test/mac-manifest.json b/render-test/mac-manifest.json
new file mode 100644
index 0000000000..224df81298
--- /dev/null
+++ b/render-test/mac-manifest.json
@@ -0,0 +1,7 @@
+{
+ "base_test_path":"../mapbox-gl-js/test/integration",
+ "expectation_paths":["expected/render-tests", "tests/mac"],
+ "ignore_paths":["../platform/node/test/ignores.json", "../render-test/mac-ignores.json", "../render-test/tests/should-fail.json"],
+ "vendor_path":"../vendor",
+ "asset_path": "../mapbox-gl-js/test/integration"
+} \ No newline at end of file
diff --git a/render-test/mac-probe-manifest.json b/render-test/mac-probe-manifest.json
new file mode 100644
index 0000000000..f3cc56d0a8
--- /dev/null
+++ b/render-test/mac-probe-manifest.json
@@ -0,0 +1,7 @@
+{
+ "probe_test_path":".",
+ "expectation_paths":["expected/render-tests", "tests/mac"],
+ "ignore_paths":["../render-test/mac-ignores.json", "../render-test/tests/should-fail.json"],
+ "vendor_path":"../vendor",
+ "asset_path": "../mapbox-gl-js/test/integration"
+} \ No newline at end of file
diff --git a/render-test/manifest_parser.cpp b/render-test/manifest_parser.cpp
new file mode 100644
index 0000000000..f1884634e5
--- /dev/null
+++ b/render-test/manifest_parser.cpp
@@ -0,0 +1,400 @@
+#include "manifest_parser.hpp"
+#include "filesystem.hpp"
+#include "parser.hpp"
+
+#include <mbgl/util/logging.hpp>
+
+#include <algorithm>
+#include <random>
+
+namespace {
+std::string removeURLArguments(const std::string& url) {
+ std::string::size_type index = url.find('?');
+ if (index != std::string::npos) {
+ return url.substr(0, index);
+ }
+ return url;
+}
+
+std::string prependFileScheme(const std::string& url) {
+ static const std::string fileScheme("file://");
+ return fileScheme + url;
+}
+} // namespace
+
+Manifest::Manifest() = default;
+Manifest::~Manifest() = default;
+
+const std::vector<TestPaths>& Manifest::getTestPaths() const {
+ return testPaths;
+}
+const std::vector<std::pair<std::string, std::string>>& Manifest::getIgnores() const {
+ return ignores;
+}
+const std::string& Manifest::getTestRootPath() const {
+ return testRootPath;
+}
+const std::string& Manifest::getManifestPath() const {
+ return manifestPath;
+}
+
+void Manifest::doShuffle(uint32_t seed) {
+ std::seed_seq sequence{seed};
+ std::mt19937 shuffler(sequence);
+ std::shuffle(testPaths.begin(), testPaths.end(), shuffler);
+}
+
+std::string Manifest::localizeURL(const std::string& url) const {
+ static const std::regex regex{"local://"};
+ if (auto vendorPath = getVendorPath(url, regex)) {
+ return *vendorPath;
+ }
+ return getIntegrationPath(url, "", regex).value_or(url);
+}
+
+void Manifest::localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const {
+ if (root.HasMember("urls") && root["urls"].IsArray()) {
+ for (auto& urlValue : root["urls"].GetArray()) {
+ const std::string path =
+ prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
+ .value_or(localizeLocalURL(urlValue.GetString()).value_or(urlValue.GetString())));
+ urlValue.Set<std::string>(path, document.GetAllocator());
+ }
+ }
+
+ if (root.HasMember("url")) {
+ static const std::string image("image");
+ static const std::string video("video");
+
+ mbgl::JSValue& urlValue = root["url"];
+ const std::string path =
+ prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
+ .value_or(localizeLocalURL(urlValue.GetString()).value_or(urlValue.GetString())));
+ urlValue.Set<std::string>(path, document.GetAllocator());
+
+ if (root["type"].GetString() != image && root["type"].GetString() != video) {
+ const auto tilesetPath = std::string(urlValue.GetString()).erase(0u, 7u); // remove "file://"
+ auto maybeTileset = readJson(tilesetPath);
+ if (maybeTileset.is<mbgl::JSDocument>()) {
+ const auto& tileset = maybeTileset.get<mbgl::JSDocument>();
+ assert(tileset.HasMember("tiles"));
+ root.AddMember("tiles", (mbgl::JSValue&)tileset["tiles"], document.GetAllocator());
+ root.RemoveMember("url");
+ }
+ }
+ }
+
+ if (root.HasMember("tiles")) {
+ mbgl::JSValue& tilesValue = root["tiles"];
+ assert(tilesValue.IsArray());
+ for (auto& tileValue : tilesValue.GetArray()) {
+ const std::string path = prependFileScheme(
+ localizeMapboxTilesURL(tileValue.GetString())
+ .value_or(localizeLocalURL(tileValue.GetString())
+ .value_or(localizeHttpURL(tileValue.GetString()).value_or(tileValue.GetString()))));
+ tileValue.Set<std::string>(path, document.GetAllocator());
+ }
+ }
+
+ if (root.HasMember("data") && root["data"].IsString()) {
+ mbgl::JSValue& dataValue = root["data"];
+ const std::string path =
+ prependFileScheme(localizeLocalURL(dataValue.GetString()).value_or(dataValue.GetString()));
+ dataValue.Set<std::string>(path, document.GetAllocator());
+ }
+}
+
+void Manifest::localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const {
+ if (root.HasMember("sources")) {
+ mbgl::JSValue& sourcesValue = root["sources"];
+ for (auto& sourceProperty : sourcesValue.GetObject()) {
+ localizeSourceURLs(sourceProperty.value, document);
+ }
+ }
+
+ if (root.HasMember("glyphs")) {
+ mbgl::JSValue& glyphsValue = root["glyphs"];
+ const std::string path = prependFileScheme(
+ localizeMapboxFontsURL(glyphsValue.GetString())
+ .value_or(localizeLocalURL(glyphsValue.GetString(), true).value_or(glyphsValue.GetString())));
+ glyphsValue.Set<std::string>(path, document.GetAllocator());
+ }
+
+ if (root.HasMember("sprite")) {
+ mbgl::JSValue& spriteValue = root["sprite"];
+ const std::string path = prependFileScheme(
+ localizeMapboxSpriteURL(spriteValue.GetString())
+ .value_or(localizeLocalURL(spriteValue.GetString()).value_or(spriteValue.GetString())));
+ spriteValue.Set<std::string>(path, document.GetAllocator());
+ }
+}
+
+mbgl::optional<std::string> Manifest::localizeLocalURL(const std::string& url, bool glyphsPath) const {
+ static const std::regex regex{"local://"};
+ if (auto vendorPath = getVendorPath(url, regex, glyphsPath)) {
+ return vendorPath;
+ }
+ return getIntegrationPath(url, "", regex, glyphsPath);
+}
+
+mbgl::optional<std::string> Manifest::localizeHttpURL(const std::string& url) const {
+ static const std::regex regex{"http://localhost:2900"};
+ if (auto vendorPath = getVendorPath(url, regex)) {
+ return vendorPath;
+ }
+ return getIntegrationPath(url, "", regex);
+}
+
+mbgl::optional<std::string> Manifest::localizeMapboxSpriteURL(const std::string& url) const {
+ static const std::regex regex{"mapbox://"};
+ return getIntegrationPath(url, "", regex);
+}
+
+mbgl::optional<std::string> Manifest::localizeMapboxFontsURL(const std::string& url) const {
+ static const std::regex regex{"mapbox://fonts"};
+ return getIntegrationPath(url, "glyphs/", regex, true);
+}
+
+mbgl::optional<std::string> Manifest::localizeMapboxTilesURL(const std::string& url) const {
+ static const std::regex regex{"mapbox://"};
+ if (auto vendorPath = getVendorPath(url, regex)) {
+ return vendorPath;
+ }
+ return getIntegrationPath(url, "tiles/", regex);
+}
+
+mbgl::optional<std::string> Manifest::localizeMapboxTilesetURL(const std::string& url) const {
+ static const std::regex regex{"mapbox://"};
+ return getIntegrationPath(url, "tilesets/", regex);
+}
+
+mbgl::optional<std::string> Manifest::getVendorPath(const std::string& url,
+ const std::regex& regex,
+ bool glyphsPath) const {
+ mbgl::filesystem::path file = std::regex_replace(url, regex, vendorPath);
+ if (mbgl::filesystem::exists(file.parent_path())) {
+ return removeURLArguments(file.string());
+ }
+
+ if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
+ return removeURLArguments(file.string());
+ }
+
+ return mbgl::nullopt;
+}
+
+mbgl::optional<std::string> Manifest::getIntegrationPath(const std::string& url,
+ const std::string& parent,
+ const std::regex& regex,
+ bool glyphsPath) const {
+ mbgl::filesystem::path file = std::regex_replace(url, regex, assetPath + parent);
+ if (mbgl::filesystem::exists(file.parent_path())) {
+ return removeURLArguments(file.string());
+ }
+
+ if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
+ return removeURLArguments(file.string());
+ }
+
+ return mbgl::nullopt;
+}
+
+namespace {
+std::vector<std::pair<std::string, std::string>> parseIgnores(const std::vector<mbgl::filesystem::path>& ignoresPaths) {
+ std::vector<std::pair<std::string, std::string>> ignores;
+ for (const auto& path : ignoresPaths) {
+ auto maybeIgnores = readJson(path);
+ if (!maybeIgnores.is<mbgl::JSDocument>()) {
+ continue;
+ }
+ for (const auto& property : maybeIgnores.get<mbgl::JSDocument>().GetObject()) {
+ const std::string ignore = {property.name.GetString(), property.name.GetStringLength()};
+ const std::string reason = {property.value.GetString(), property.value.GetStringLength()};
+ ignores.emplace_back(std::make_pair(ignore, reason));
+ }
+ }
+
+ return ignores;
+}
+
+std::vector<mbgl::filesystem::path> getTestExpectations(mbgl::filesystem::path testPath,
+ const mbgl::filesystem::path& testsRootPath,
+ std::vector<mbgl::filesystem::path> expectationsPaths) {
+ std::vector<mbgl::filesystem::path> expectations{std::move(testPath.remove_filename())};
+ const auto& defaultTestExpectationsPath = expectations.front().string();
+
+ const std::regex regex{testsRootPath.string()};
+ for (const auto& path : expectationsPaths) {
+ expectations.emplace_back(std::regex_replace(defaultTestExpectationsPath, regex, path.string()));
+ assert(!expectations.back().empty());
+ }
+
+ return expectations;
+}
+
+mbgl::filesystem::path getValidPath(const std::string& manifestPath, const std::string& path) {
+ const static mbgl::filesystem::path BasePath{manifestPath};
+ mbgl::filesystem::path result{path};
+ if (result.is_relative()) {
+ result = BasePath / result;
+ }
+ if (mbgl::filesystem::exists(result)) {
+ return result;
+ }
+ mbgl::Log::Warning(mbgl::Event::General, "Invalid path is provoided inside the manifest file: %s", path.c_str());
+ return mbgl::filesystem::path{};
+}
+
+} // namespace
+
+mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manifestPath,
+ const std::vector<std::string>& testNames,
+ const std::string& testFilter) {
+ Manifest manifest;
+ const auto filePath = mbgl::filesystem::path(manifestPath);
+ manifest.manifestPath = manifestPath.substr(0, manifestPath.find(filePath.filename()));
+
+ auto contents = readJson(filePath);
+ if (!contents.is<mbgl::JSDocument>()) {
+ mbgl::Log::Error(mbgl::Event::General, "Provided manifest file: %s is not a valid json", filePath.c_str());
+ return mbgl::nullopt;
+ }
+
+ auto document = std::move(contents.get<mbgl::JSDocument>());
+ if (document.HasMember("asset_path")) {
+ const auto& assetPathValue = document["asset_path"];
+ if (!assetPathValue.IsString()) {
+ mbgl::Log::Warning(
+ mbgl::Event::General, "Invalid assetPath is provoided inside the manifest file: %s", filePath.c_str());
+ return mbgl::nullopt;
+ }
+ manifest.assetPath = (getValidPath(manifest.manifestPath, assetPathValue.GetString()) / "").string();
+ if (manifest.assetPath.empty()) {
+ return mbgl::nullopt;
+ }
+ }
+ if (document.HasMember("vendor_path")) {
+ const auto& vendorPathValue = document["vendor_path"];
+ if (!vendorPathValue.IsString()) {
+ mbgl::Log::Warning(
+ mbgl::Event::General, "Invalid vendorPath is provoided inside the manifest file: %s", filePath.c_str());
+ return mbgl::nullopt;
+ }
+ manifest.vendorPath = (getValidPath(manifest.manifestPath, vendorPathValue.GetString()) / "").string();
+ if (manifest.vendorPath.empty()) {
+ return mbgl::nullopt;
+ }
+ }
+ mbgl::filesystem::path baseTestPath;
+ if (document.HasMember("base_test_path")) {
+ const auto& testPathValue = document["base_test_path"];
+ if (!testPathValue.IsString()) {
+ mbgl::Log::Warning(
+ mbgl::Event::General, "Invalid testPath is provoided inside the manifest file: %s", filePath.c_str());
+ return mbgl::nullopt;
+ }
+ baseTestPath = getValidPath(manifest.manifestPath, testPathValue.GetString());
+ if (baseTestPath.empty()) {
+ return mbgl::nullopt;
+ }
+ }
+ mbgl::filesystem::path probeTestPath;
+ bool enbaleProbeTest{false};
+ if (document.HasMember("probe_test_path")) {
+ const auto& testPathValue = document["probe_test_path"];
+ if (!testPathValue.IsString()) {
+ mbgl::Log::Warning(
+ mbgl::Event::General, "Invalid testPath is provoided inside the manifest file: %s", filePath.c_str());
+ return mbgl::nullopt;
+ }
+ probeTestPath = getValidPath(manifest.manifestPath, testPathValue.GetString());
+ if (probeTestPath.empty()) {
+ return mbgl::nullopt;
+ }
+ enbaleProbeTest = true;
+ }
+ std::vector<mbgl::filesystem::path> expectationPaths{};
+ if (document.HasMember("expectation_paths")) {
+ const auto& expectationPathValue = document["expectation_paths"];
+ if (!expectationPathValue.IsArray()) {
+ mbgl::Log::Warning(mbgl::Event::General,
+ "Provided expectation_paths inside the manifest file: %s is not a valid array",
+ filePath.c_str());
+ return mbgl::nullopt;
+ }
+ for (const auto& value : expectationPathValue.GetArray()) {
+ if (!value.IsString()) {
+ mbgl::Log::Warning(mbgl::Event::General,
+ "Invalid expectation path item is provoided inside the manifest file: %s",
+ filePath.c_str());
+ return mbgl::nullopt;
+ }
+ expectationPaths.emplace_back(getValidPath(manifest.manifestPath, value.GetString()));
+ if (expectationPaths.back().empty()) {
+ return mbgl::nullopt;
+ }
+ }
+ }
+ std::vector<mbgl::filesystem::path> ignorePaths{};
+ if (document.HasMember("ignore_paths")) {
+ const auto& ignorePathValue = document["ignore_paths"];
+ if (!ignorePathValue.IsArray()) {
+ mbgl::Log::Warning(mbgl::Event::General,
+ "Provided ignore_paths inside the manifest file: %s is not a valid array",
+ filePath.c_str());
+ return mbgl::nullopt;
+ }
+ for (const auto& value : ignorePathValue.GetArray()) {
+ if (!value.IsString()) {
+ mbgl::Log::Warning(mbgl::Event::General,
+ "Invalid ignore path item is provoided inside the manifest file: %s",
+ filePath.c_str());
+ return mbgl::nullopt;
+ }
+ ignorePaths.emplace_back(getValidPath(manifest.manifestPath, value.GetString()));
+ if (ignorePaths.back().empty()) {
+ return mbgl::nullopt;
+ }
+ }
+ manifest.ignores = parseIgnores(ignorePaths);
+ }
+
+ manifest.testRootPath = enbaleProbeTest ? probeTestPath.string() : baseTestPath.string();
+ if (manifest.testRootPath.back() == '/') {
+ manifest.testRootPath.pop_back();
+ }
+ if (manifest.manifestPath.back() == '/') {
+ manifest.manifestPath.pop_back();
+ }
+
+ std::vector<mbgl::filesystem::path> paths;
+ for (const auto& id : testNames) {
+ paths.emplace_back(manifest.testRootPath + "/" + id);
+ }
+ if (paths.empty()) {
+ paths.emplace_back(manifest.testRootPath);
+ }
+
+ // Recursively traverse through the test paths and collect test directories containing "style.json".
+ auto& testPaths = manifest.testPaths;
+ testPaths.reserve(paths.size());
+ for (const auto& path : paths) {
+ if (!mbgl::filesystem::exists(path)) {
+ mbgl::Log::Warning(
+ mbgl::Event::General, "Provided test folder '%s' does not exist.", path.string().c_str());
+ continue;
+ }
+ for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(path)) {
+ // Skip paths that fail regexp match.
+ if (!testFilter.empty() && !std::regex_match(testPath.path().string(), std::regex(testFilter))) {
+ continue;
+ }
+
+ if (testPath.path().filename() == "style.json") {
+ testPaths.emplace_back(testPath, getTestExpectations(testPath, path, expectationPaths));
+ }
+ }
+ }
+
+ return mbgl::optional<Manifest>(manifest);
+}
diff --git a/render-test/manifest_parser.hpp b/render-test/manifest_parser.hpp
new file mode 100644
index 0000000000..bc5adf1091
--- /dev/null
+++ b/render-test/manifest_parser.hpp
@@ -0,0 +1,55 @@
+#pragma once
+
+#include "metadata.hpp"
+
+#include <mbgl/util/optional.hpp>
+#include <mbgl/util/rapidjson.hpp>
+
+#include <regex>
+#include <string>
+#include <utility>
+#include <vector>
+
+class Manifest {
+public:
+ Manifest();
+ ~Manifest();
+ const std::vector<std::pair<std::string, std::string>>& getIgnores() const;
+ const std::vector<TestPaths>& getTestPaths() const;
+ const std::string& getTestRootPath() const;
+ const std::string& getManifestPath() const;
+ void doShuffle(uint32_t seed);
+
+ std::string localizeURL(const std::string& url) const;
+ void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const;
+ void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const;
+
+private:
+ friend class ManifestParser;
+ mbgl::optional<std::string> localizeLocalURL(const std::string& url, bool glyphsPath = false) const;
+ mbgl::optional<std::string> localizeHttpURL(const std::string& url) const;
+ mbgl::optional<std::string> localizeMapboxSpriteURL(const std::string& url) const;
+ mbgl::optional<std::string> localizeMapboxFontsURL(const std::string& url) const;
+ mbgl::optional<std::string> localizeMapboxTilesURL(const std::string& url) const;
+ mbgl::optional<std::string> localizeMapboxTilesetURL(const std::string& url) const;
+ mbgl::optional<std::string> getVendorPath(const std::string& url,
+ const std::regex& regex,
+ bool glyphsPath = false) const;
+ mbgl::optional<std::string> getIntegrationPath(const std::string& url,
+ const std::string& parent,
+ const std::regex& regex,
+ bool glyphsPath = false) const;
+ std::string manifestPath;
+ std::string testRootPath;
+ std::string vendorPath;
+ std::string assetPath;
+ std::vector<std::pair<std::string, std::string>> ignores;
+ std::vector<TestPaths> testPaths;
+};
+
+class ManifestParser {
+public:
+ static mbgl::optional<Manifest> parseManifest(const std::string& manifestPath,
+ const std::vector<std::string>& testNames,
+ const std::string& testFilter);
+};
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index 11a42f3202..5a91fc7a58 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -3,8 +3,6 @@
#include <mbgl/util/rapidjson.hpp>
#include <mbgl/util/string.hpp>
-#include <args.hxx>
-
#include <rapidjson/prettywriter.h>
#include <rapidjson/stringbuffer.h>
#include <rapidjson/writer.h>
@@ -23,8 +21,8 @@
#include "parser.hpp"
#include "runner.hpp"
-#include <sstream>
#include <regex>
+#include <sstream>
namespace {
@@ -83,101 +81,6 @@ const char* resultsHeaderButtons = R"HTML(
</h1>
)HTML";
-std::string removeURLArguments(const std::string &url) {
- std::string::size_type index = url.find('?');
- if (index != std::string::npos) {
- return url.substr(0, index);
- }
- return url;
-}
-
-std::string prependFileScheme(const std::string &url) {
- static const std::string fileScheme("file://");
- return fileScheme + url;
-}
-
-mbgl::optional<std::string> getVendorPath(const std::string& url,
- const std::regex& regex,
- const std::string& testRootPath,
- bool glyphsPath = false) {
- static const mbgl::filesystem::path vendorPath = getValidPath(testRootPath, std::string("vendor/"));
-
- mbgl::filesystem::path file = std::regex_replace(url, regex, vendorPath.string());
- if (mbgl::filesystem::exists(file.parent_path())) {
- return removeURLArguments(file.string());
- }
-
- if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
- return removeURLArguments(file.string());
- }
-
- return {};
-}
-
-mbgl::optional<std::string> getIntegrationPath(const std::string& url,
- const std::string& parent,
- const std::regex& regex,
- const std::string& testRootPath,
- bool glyphsPath = false) {
- static const mbgl::filesystem::path integrationPath =
- getValidPath(testRootPath, std::string("mapbox-gl-js/test/integration/"));
-
- mbgl::filesystem::path file = std::regex_replace(url, regex, integrationPath.string() + parent);
- if (mbgl::filesystem::exists(file.parent_path())) {
- return removeURLArguments(file.string());
- }
-
- if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
- return removeURLArguments(file.string());
- }
-
- return {};
-}
-
-mbgl::optional<std::string> localizeLocalURL(const std::string& url,
- const std::string& testRootPath,
- bool glyphsPath = false) {
- static const std::regex regex{"local://"};
- if (auto vendorPath = getVendorPath(url, regex, testRootPath, glyphsPath)) {
- return vendorPath;
- } else {
- return getIntegrationPath(url, "", regex, testRootPath, glyphsPath);
- }
-}
-
-mbgl::optional<std::string> localizeHttpURL(const std::string& url, const std::string& testRootPath) {
- static const std::regex regex{"http://localhost:2900"};
- if (auto vendorPath = getVendorPath(url, regex, testRootPath)) {
- return vendorPath;
- } else {
- return getIntegrationPath(url, "", regex, testRootPath);
- }
-}
-
-mbgl::optional<std::string> localizeMapboxSpriteURL(const std::string& url, const std::string& testRootPath) {
- static const std::regex regex{"mapbox://"};
- return getIntegrationPath(url, "", regex, testRootPath);
-}
-
-mbgl::optional<std::string> localizeMapboxFontsURL(const std::string& url, const std::string& testRootPath) {
- static const std::regex regex{"mapbox://fonts"};
- return getIntegrationPath(url, "glyphs/", regex, testRootPath, true);
-}
-
-mbgl::optional<std::string> localizeMapboxTilesURL(const std::string& url, const std::string& testRootPath) {
- static const std::regex regex{"mapbox://"};
- if (auto vendorPath = getVendorPath(url, regex, testRootPath)) {
- return vendorPath;
- } else {
- return getIntegrationPath(url, "tiles/", regex, testRootPath);
- }
-}
-
-mbgl::optional<std::string> localizeMapboxTilesetURL(const std::string& url, const std::string& testRootPath) {
- static const std::regex regex{"mapbox://"};
- return getIntegrationPath(url, "tilesets/", regex, testRootPath);
-}
-
void writeJSON(rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer, const mbgl::Value& value) {
value.match([&writer](const mbgl::NullValue&) { writer.Null(); },
[&writer](bool b) { writer.Bool(b); },
@@ -205,33 +108,6 @@ void writeJSON(rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer, const m
} // namespace
-static const mbgl::filesystem::path DefaultRootPath{std::string(TEST_RUNNER_ROOT_PATH)};
-
-const mbgl::filesystem::path getValidPath(const std::string& basePath, const std::string& subPath) {
- auto filePath = mbgl::filesystem::path(basePath) / subPath;
- if (mbgl::filesystem::exists(filePath)) {
- return filePath;
- }
- // Fall back to check default path
- filePath = DefaultRootPath / subPath;
- if (mbgl::filesystem::exists(filePath)) {
- return filePath;
- }
- mbgl::Log::Warning(mbgl::Event::General, "Failed to find path: %s", subPath.c_str());
- return mbgl::filesystem::path{};
-}
-
-/// Returns path of the render test cases directory.
-const std::string getTestPath(const std::string& rootTestPath) {
- // Check if sub-directory exits or not
- auto testBasePath = mbgl::filesystem::path(rootTestPath) / ("mapbox-gl-js/test/integration");
- if (mbgl::filesystem::exists(testBasePath)) {
- return testBasePath.string();
- }
- // Use root test path for further processing
- return rootTestPath;
-}
-
std::string toJSON(const mbgl::Value& value, unsigned indent, bool singleLine) {
rapidjson::StringBuffer buffer;
rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
@@ -284,7 +160,7 @@ JSONReply readJson(const mbgl::filesystem::path& jsonPath) {
return { mbgl::formatJSONParseError(document) };
}
- return { std::move(document) };
+ return {std::move(document)};
}
std::string serializeJsonValue(const mbgl::JSValue& value) {
@@ -392,170 +268,6 @@ std::vector<std::string> readExpectedJSONEntries(const mbgl::filesystem::path& b
return readExpectedEntries(regex, base);
}
-namespace {
-
-std::vector<mbgl::filesystem::path> getTestExpectations(mbgl::filesystem::path testPath,
- const mbgl::filesystem::path& testsRootPath,
- std::vector<mbgl::filesystem::path> expectationsPaths) {
- std::vector<mbgl::filesystem::path> expectations{std::move(testPath.remove_filename())};
- const auto& defaultTestExpectationsPath = expectations.front().string();
-
- const std::regex regex{testsRootPath.string()};
- for (const auto& path : expectationsPaths) {
- expectations.emplace_back(std::regex_replace(defaultTestExpectationsPath, regex, path.string()));
- assert(!expectations.back().empty());
- }
-
- return expectations;
-}
-
-} // namespace
-
-ArgumentsTuple parseArguments(int argc, char** argv) {
- args::ArgumentParser argumentParser("Mapbox GL Test Runner");
-
- args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", { 'h', "help" });
-
- args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", {'r', "recycle-map"});
- args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order", {'s', "shuffle"});
- args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)",
- { "seed" });
- args::ValueFlag<std::string> testPathValue(argumentParser, "rootPath", "Test root rootPath", {'p', "rootPath"});
- args::ValueFlag<std::regex> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"});
- args::ValueFlag<std::string> expectationsPathValue(
- argumentParser, "expectationsPath", "Test expectations path", {'e', "expectationsPath"});
- args::ValueFlag<std::string> ignoresPathValue(
- argumentParser, "ignoresPath", "Test ignore list path", {'i', "ignoresPath"});
- args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
-
- try {
- argumentParser.ParseCLI(argc, argv);
- } catch (const args::Help&) {
- std::ostringstream stream;
- stream << argumentParser;
- mbgl::Log::Info(mbgl::Event::General, stream.str());
- exit(0);
- } catch (const args::ParseError& e) {
- std::ostringstream stream;
- stream << argumentParser;
- mbgl::Log::Info(mbgl::Event::General, stream.str());
- mbgl::Log::Error(mbgl::Event::General, e.what());
- exit(1);
- } catch (const args::ValidationError& e) {
- std::ostringstream stream;
- stream << argumentParser;
- mbgl::Log::Info(mbgl::Event::General, stream.str());
- mbgl::Log::Error(mbgl::Event::General, e.what());
- exit(2);
- } catch (const std::regex_error& e) {
- mbgl::Log::Error(mbgl::Event::General, "Invalid filter regular expression: %s", e.what());
- exit(3);
- }
-
- const auto testRootPath = testPathValue ? args::get(testPathValue) : std::string{TEST_RUNNER_ROOT_PATH};
- mbgl::filesystem::path rootPath{testRootPath};
- if (!mbgl::filesystem::exists(rootPath)) {
- mbgl::Log::Error(
- mbgl::Event::General, "Provided test rootPath '%s' does not exist.", rootPath.string().c_str());
- exit(4);
- }
- std::vector<mbgl::filesystem::path> expectationsPaths;
- if (expectationsPathValue) {
- auto expectationsPath = mbgl::filesystem::path(testRootPath) / args::get(expectationsPathValue);
- if (!mbgl::filesystem::exists(expectationsPath)) {
- mbgl::Log::Error(mbgl::Event::General,
- "Provided expectationsPath '%s' does not exist.",
- expectationsPath.string().c_str());
- exit(5);
- }
- expectationsPaths.emplace_back(std::move(expectationsPath));
- }
-
- std::string ignoresPath{};
- if (ignoresPathValue) {
- auto path = mbgl::filesystem::path(testRootPath) / args::get(ignoresPathValue);
- if (!mbgl::filesystem::exists(path)) {
- mbgl::Log::Error(
- mbgl::Event::General, "Provided ignore list path '%s' does not exist.", path.string().c_str());
- exit(6);
- }
- ignoresPath = path.string();
- }
-
- std::vector<mbgl::filesystem::path> paths;
- auto testBasePath = mbgl::filesystem::path(getTestPath(testRootPath));
- for (const auto& id : args::get(testNameValues)) {
- paths.emplace_back(testBasePath / id);
- }
-
- if (paths.empty()) {
- paths.emplace_back(testBasePath);
- }
-
- // Recursively traverse through the test paths and collect test directories containing "style.json".
- std::vector<TestPaths> testPaths;
- testPaths.reserve(paths.size());
- for (const auto& path : paths) {
- if (!mbgl::filesystem::exists(path)) {
- mbgl::Log::Warning(mbgl::Event::General, "Provided test folder '%s' does not exist.", path.string().c_str());
- continue;
- }
- for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(path)) {
- // Skip paths that fail regexp match.
- if (testFilterValue && !std::regex_match(testPath.path().string(), args::get(testFilterValue))) {
- continue;
- }
-
- if (testPath.path().filename() == "style.json") {
- testPaths.emplace_back(testPath, getTestExpectations(testPath, path, expectationsPaths));
- }
- }
- }
-
- return ArgumentsTuple{recycleMapFlag ? args::get(recycleMapFlag) : false,
- shuffleFlag ? args::get(shuffleFlag) : false,
- seedValue ? args::get(seedValue) : 1u,
- testRootPath,
- ignoresPath,
- std::move(testPaths)};
-}
-
-std::vector<std::pair<std::string, std::string>> parseIgnores(const std::string& testRootPath,
- const std::string& ignoresPath) {
- std::vector<std::pair<std::string, std::string>> ignores;
- auto mainIgnoresPath = getValidPath(testRootPath, "platform/node/test/ignores.json");
-
- mbgl::filesystem::path platformSpecificIgnores;
- mbgl::filesystem::path ownTestsIgnores = getValidPath(testRootPath, "render-test/tests/should-fail.json");
-
-#ifdef __APPLE__
- platformSpecificIgnores = getValidPath(testRootPath, "render-test/mac-ignores.json");
-#elif __linux__
- platformSpecificIgnores = getValidPath(testRootPath, "render-test/linux-ignores.json");
-#endif
-
- std::vector<mbgl::filesystem::path> ignoresPaths = {mainIgnoresPath, platformSpecificIgnores, ownTestsIgnores};
-
- if (!ignoresPath.empty()) {
- ignoresPaths.emplace_back(ignoresPath);
- }
- for (const auto& path : ignoresPaths) {
- auto maybeIgnores = readJson(path);
- if (!maybeIgnores.is<mbgl::JSDocument>()) {
- continue;
- }
- for (const auto& property : maybeIgnores.get<mbgl::JSDocument>().GetObject()) {
- const std::string ignore = { property.name.GetString(),
- property.name.GetStringLength() };
- const std::string reason = { property.value.GetString(),
- property.value.GetStringLength() };
- ignores.emplace_back(std::make_pair(ignore, reason));
- }
- }
-
- return ignores;
-}
-
TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
TestMetrics result;
@@ -601,7 +313,7 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()};
assert(!mark.empty());
result.memory.emplace(std::piecewise_construct,
- std::forward_as_tuple(std::move(mark)),
+ std::forward_as_tuple(std::move(mark)),
std::forward_as_tuple(probeValue[1].GetUint64(), probeValue[2].GetUint64()));
}
}
@@ -645,7 +357,7 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
return result;
}
-TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRootPath) {
+TestMetadata parseTestMetadata(const TestPaths& paths, const Manifest& manifest) {
TestMetadata metadata;
metadata.paths = paths;
@@ -656,7 +368,7 @@ TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRo
}
metadata.document = std::move(maybeJson.get<mbgl::JSDocument>());
- localizeStyleURLs(metadata.document, metadata.document, testRootPath);
+ manifest.localizeStyleURLs(metadata.document, metadata.document);
if (!metadata.document.HasMember("metadata")) {
mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s", paths.stylePath.c_str());
@@ -665,8 +377,7 @@ TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRo
const mbgl::JSValue& metadataValue = metadata.document["metadata"];
if (!metadataValue.HasMember("test")) {
- mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s",
- paths.stylePath.c_str());
+ mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s", paths.stylePath.c_str());
return metadata;
}
@@ -694,8 +405,8 @@ TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRo
if (testValue.HasMember("description")) {
assert(testValue["description"].IsString());
- metadata.description = std::string{ testValue["description"].GetString(),
- testValue["description"].GetStringLength() };
+ metadata.description =
+ std::string{testValue["description"].GetString(), testValue["description"].GetStringLength()};
}
if (testValue.HasMember("mapMode")) {
@@ -802,8 +513,9 @@ std::string encodeBase64(const std::string& data) {
}
std::string createResultItem(const TestMetadata& metadata, bool hasFailedTests) {
- const bool shouldHide = (hasFailedTests && metadata.status == "passed") || (metadata.status.find("ignored") != std::string::npos);
-
+ const bool shouldHide =
+ (hasFailedTests && metadata.status == "passed") || (metadata.status.find("ignored") != std::string::npos);
+
std::string html;
html.append("<div class=\"test " + metadata.status + (shouldHide ? " hide" : "") + "\">\n");
html.append(R"(<h2><span class="label" style="background: )" + metadata.color + "\">" + metadata.status + "</span> " + metadata.id + "</h2>\n");
@@ -918,91 +630,3 @@ std::string createResultPage(const TestStatistics& stats, const std::vector<Test
return resultsPage;
}
-
-std::string localizeURL(const std::string& url, const std::string& testRootPath) {
- static const std::regex regex{"local://"};
- if (auto vendorPath = getVendorPath(url, regex, testRootPath)) {
- return *vendorPath;
- } else {
- return getIntegrationPath(url, "", regex, testRootPath).value_or(url);
- }
-}
-
-void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const std::string& testRootPath) {
- if (root.HasMember("urls") && root["urls"].IsArray()) {
- for (auto& urlValue : root["urls"].GetArray()) {
- const std::string path = prependFileScheme(
- localizeMapboxTilesetURL(urlValue.GetString(), testRootPath)
- .value_or(localizeLocalURL(urlValue.GetString(), testRootPath).value_or(urlValue.GetString())));
- urlValue.Set<std::string>(path, document.GetAllocator());
- }
- }
-
- if (root.HasMember("url")) {
- static const std::string image("image");
- static const std::string video("video");
-
- mbgl::JSValue& urlValue = root["url"];
- const std::string path = prependFileScheme(
- localizeMapboxTilesetURL(urlValue.GetString(), testRootPath)
- .value_or(localizeLocalURL(urlValue.GetString(), testRootPath).value_or(urlValue.GetString())));
- urlValue.Set<std::string>(path, document.GetAllocator());
-
- if (root["type"].GetString() != image && root["type"].GetString() != video) {
- const auto tilesetPath = std::string(urlValue.GetString()).erase(0u, 7u); // remove "file://"
- auto maybeTileset = readJson(tilesetPath);
- if (maybeTileset.is<mbgl::JSDocument>()) {
- const auto& tileset = maybeTileset.get<mbgl::JSDocument>();
- assert(tileset.HasMember("tiles"));
- root.AddMember("tiles", (mbgl::JSValue&)tileset["tiles"], document.GetAllocator());
- root.RemoveMember("url");
- }
- }
- }
-
- if (root.HasMember("tiles")) {
- mbgl::JSValue& tilesValue = root["tiles"];
- assert(tilesValue.IsArray());
- for (auto& tileValue : tilesValue.GetArray()) {
- const std::string path =
- prependFileScheme(localizeMapboxTilesURL(tileValue.GetString(), testRootPath)
- .value_or(localizeLocalURL(tileValue.GetString(), testRootPath)
- .value_or(localizeHttpURL(tileValue.GetString(), testRootPath)
- .value_or(tileValue.GetString()))));
- tileValue.Set<std::string>(path, document.GetAllocator());
- }
- }
-
- if (root.HasMember("data") && root["data"].IsString()) {
- mbgl::JSValue& dataValue = root["data"];
- const std::string path =
- prependFileScheme(localizeLocalURL(dataValue.GetString(), testRootPath).value_or(dataValue.GetString()));
- dataValue.Set<std::string>(path, document.GetAllocator());
- }
-}
-
-void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const std::string& testRootPath) {
- if (root.HasMember("sources")) {
- mbgl::JSValue& sourcesValue = root["sources"];
- for (auto& sourceProperty : sourcesValue.GetObject()) {
- localizeSourceURLs(sourceProperty.value, document, testRootPath);
- }
- }
-
- if (root.HasMember("glyphs")) {
- mbgl::JSValue& glyphsValue = root["glyphs"];
- const std::string path = prependFileScheme(
- localizeMapboxFontsURL(glyphsValue.GetString(), testRootPath)
- .value_or(
- localizeLocalURL(glyphsValue.GetString(), testRootPath, true).value_or(glyphsValue.GetString())));
- glyphsValue.Set<std::string>(path, document.GetAllocator());
- }
-
- if (root.HasMember("sprite")) {
- mbgl::JSValue& spriteValue = root["sprite"];
- const std::string path = prependFileScheme(
- localizeMapboxSpriteURL(spriteValue.GetString(), testRootPath)
- .value_or(localizeLocalURL(spriteValue.GetString(), testRootPath).value_or(spriteValue.GetString())));
- spriteValue.Set<std::string>(path, document.GetAllocator());
- }
-}
diff --git a/render-test/parser.hpp b/render-test/parser.hpp
index 1985b9cffc..3d79ac668a 100644
--- a/render-test/parser.hpp
+++ b/render-test/parser.hpp
@@ -5,15 +5,15 @@
#include <mbgl/util/rapidjson.hpp>
#include <mbgl/util/variant.hpp>
-#include <tuple>
#include <string>
+#include <tuple>
#include <vector>
+class Manifest;
+
using ErrorMessage = std::string;
using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
-using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::string, std::vector<TestPaths>>;
-
JSONReply readJson(const mbgl::filesystem::path&);
std::string serializeJsonValue(const mbgl::JSValue&);
std::string serializeMetrics(const TestMetrics&);
@@ -23,21 +23,9 @@ std::vector<std::string> readExpectedJSONEntries(const mbgl::filesystem::path& b
TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path);
-const std::string getTestPath(const std::string& rootTestPath);
-const mbgl::filesystem::path getValidPath(const std::string& basePath, const std::string& subPath);
-
-ArgumentsTuple parseArguments(int argc, char** argv);
-std::vector<std::pair<std::string, std::string>> parseIgnores(const std::string& testRootPath,
- const std::string& ignoresPath);
-
-TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRootPath);
+TestMetadata parseTestMetadata(const TestPaths& paths, const Manifest& manifest);
std::string createResultPage(const TestStatistics&, const std::vector<TestMetadata>&, bool shuffle, uint32_t seed);
-std::string localizeURL(const std::string& url, const std::string& testPath);
-
std::string toJSON(const mbgl::Value& value, unsigned indent, bool singleLine);
std::string toJSON(const std::vector<mbgl::Feature>& features, unsigned indent, bool singleLine);
-
-void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const std::string& testPath);
-void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const std::string& testPath);
diff --git a/render-test/render_test.cpp b/render-test/render_test.cpp
index 2cf3f5fd65..38d6c15f3f 100644
--- a/render-test/render_test.cpp
+++ b/render-test/render_test.cpp
@@ -2,14 +2,16 @@
#include <mbgl/render_test.hpp>
#include <mbgl/util/io.hpp>
+#include <mbgl/util/logging.hpp>
#include <mbgl/util/run_loop.hpp>
+#include <args.hxx>
+
+#include "manifest_parser.hpp"
#include "metadata.hpp"
#include "parser.hpp"
#include "runner.hpp"
-#include <random>
-
#define ANSI_COLOR_RED "\x1b[31m"
#define ANSI_COLOR_GREEN "\x1b[32m"
#define ANSI_COLOR_YELLOW "\x1b[33m"
@@ -37,38 +39,97 @@ void operator delete(void* ptr, size_t) noexcept {
}
#endif
+namespace {
+using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>, std::string>;
+ArgumentsTuple parseArguments(int argc, char** argv) {
+ args::ArgumentParser argumentParser("Mapbox GL Test Runner");
+
+ args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", {'h', "help"});
+
+ args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", {'r', "recycle-map"});
+ args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order", {'s', "shuffle"});
+ args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)", {"seed"});
+ args::ValueFlag<std::string> testPathValue(
+ argumentParser, "manifestPath", "Test manifest file path", {'p', "manifestPath"});
+ args::ValueFlag<std::string> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"});
+ args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
+
+ try {
+ argumentParser.ParseCLI(argc, argv);
+ } catch (const args::Help&) {
+ std::ostringstream stream;
+ stream << argumentParser;
+ mbgl::Log::Info(mbgl::Event::General, stream.str());
+ exit(0);
+ } catch (const args::ParseError& e) {
+ std::ostringstream stream;
+ stream << argumentParser;
+ mbgl::Log::Info(mbgl::Event::General, stream.str());
+ mbgl::Log::Error(mbgl::Event::General, e.what());
+ exit(1);
+ } catch (const args::ValidationError& e) {
+ std::ostringstream stream;
+ stream << argumentParser;
+ mbgl::Log::Info(mbgl::Event::General, stream.str());
+ mbgl::Log::Error(mbgl::Event::General, e.what());
+ exit(2);
+ } catch (const std::regex_error& e) {
+ mbgl::Log::Error(mbgl::Event::General, "Invalid filter regular expression: %s", e.what());
+ exit(3);
+ }
+
+ mbgl::filesystem::path manifestPath{testPathValue ? args::get(testPathValue) : std::string{TEST_RUNNER_ROOT_PATH}};
+ if (!mbgl::filesystem::exists(manifestPath) || !manifestPath.has_filename()) {
+ mbgl::Log::Error(mbgl::Event::General,
+ "Provided test manifest file path '%s' does not exist",
+ manifestPath.string().c_str());
+ exit(4);
+ }
+
+ auto testNames = testNameValues ? args::get(testNameValues) : std::vector<std::string>{};
+ auto testFilter = testFilterValue ? args::get(testFilterValue) : std::string{};
+ const auto shuffle = shuffleFlag ? args::get(shuffleFlag) : false;
+ const auto seed = seedValue ? args::get(seedValue) : 1u;
+ return ArgumentsTuple{recycleMapFlag ? args::get(recycleMapFlag) : false,
+ shuffle,
+ seed,
+ manifestPath.string(),
+ std::move(testNames),
+ std::move(testFilter)};
+}
+} // namespace
namespace mbgl {
int runRenderTests(int argc, char** argv) {
bool recycleMap;
bool shuffle;
uint32_t seed;
- std::string testRootPath;
- std::string ignoresPath;
- std::vector<TestPaths> testPaths;
-
- std::tie(recycleMap, shuffle, seed, testRootPath, ignoresPath, testPaths) = parseArguments(argc, argv);
-
- const auto ignores = parseIgnores(testRootPath, ignoresPath);
-
+ std::string manifestPath;
+ std::vector<std::string> testNames;
+ std::string testFilter;
+
+ std::tie(recycleMap, shuffle, seed, manifestPath, testNames, testFilter) = parseArguments(argc, argv);
+ auto manifestData = ManifestParser::parseManifest(manifestPath, testNames, testFilter);
+ if (!manifestData) {
+ exit(5);
+ }
+ mbgl::util::RunLoop runLoop;
+ TestRunner runner(std::move(*manifestData));
if (shuffle) {
printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed);
-
- std::seed_seq sequence { seed };
- std::mt19937 shuffler(sequence);
- std::shuffle(testPaths.begin(), testPaths.end(), shuffler);
+ runner.doShuffle(seed);
}
- mbgl::util::RunLoop runLoop;
- TestRunner runner(testRootPath);
-
+ const auto& manifest = runner.getManifest();
+ const auto& ignores = manifest.getIgnores();
+ const auto& testPaths = manifest.getTestPaths();
std::vector<TestMetadata> metadatas;
metadatas.reserve(testPaths.size());
TestStatistics stats;
for (auto& testPath : testPaths) {
- TestMetadata metadata = parseTestMetadata(testPath, testRootPath);
+ TestMetadata metadata = parseTestMetadata(testPath, manifest);
if (!recycleMap) {
runner.reset();
@@ -78,7 +139,7 @@ int runRenderTests(int argc, char** argv) {
std::string& status = metadata.status;
std::string& color = metadata.color;
- const std::string::size_type rootLength = getTestPath(testRootPath).length();
+ const std::string::size_type rootLength = manifest.getTestRootPath().length();
id = testPath.defaultExpectations();
id = id.substr(rootLength + 1, id.length() - rootLength - 2);
@@ -138,13 +199,14 @@ int runRenderTests(int argc, char** argv) {
metadatas.push_back(std::move(metadata));
}
-
+ const auto& testRootPath = manifest.getManifestPath();
+ const auto resultPath =
+ testRootPath + "/" + (testNames.empty() ? "render-tests" : testNames.front()) + "_index.html";
std::string resultsHTML = createResultPage(stats, metadatas, shuffle, seed);
- mbgl::util::write_file(testRootPath + "/index.html", resultsHTML);
+ mbgl::util::write_file(resultPath, resultsHTML);
- const uint32_t count = stats.erroredTests + stats.failedTests +
- stats.ignoreFailedTests + stats.ignorePassedTests +
- stats.passedTests;
+ const uint32_t count =
+ stats.erroredTests + stats.failedTests + stats.ignoreFailedTests + stats.ignorePassedTests + stats.passedTests;
if (stats.passedTests) {
printf(ANSI_COLOR_GREEN "%u passed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.passedTests, 100.0 * stats.passedTests / count);
@@ -162,7 +224,7 @@ int runRenderTests(int argc, char** argv) {
printf(ANSI_COLOR_RED "%u errored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.erroredTests, 100.0 * stats.erroredTests / count);
}
- printf("Results at: %s%s\n", testRootPath.c_str(), "/index.html");
+ printf("Results at: %s\n", resultPath.c_str());
return stats.failedTests + stats.erroredTests == 0 ? 0 : 1;
}
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 4c0967a9e9..e882e394be 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -101,7 +101,15 @@ std::string simpleDiff(const Value& result, const Value& expected) {
return diff.str();
}
-TestRunner::TestRunner(const std::string& testRootPath_) : maps(), testRootPath(testRootPath_) {}
+TestRunner::TestRunner(Manifest manifest_) : manifest(std::move(manifest_)) {}
+
+const Manifest& TestRunner::getManifest() const {
+ return manifest;
+}
+
+void TestRunner::doShuffle(uint32_t seed) {
+ manifest.doShuffle(seed);
+}
bool TestRunner::checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
std::vector<mbgl::Feature>&& features,
@@ -487,7 +495,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
std::string imagePath = operationArray[2].GetString();
imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end());
- const mbgl::filesystem::path filePath = mbgl::filesystem::path(getTestPath(testRootPath)) / imagePath;
+ const mbgl::filesystem::path filePath = mbgl::filesystem::path(manifest.getTestRootPath()) / imagePath;
mbgl::optional<std::string> maybeImage = mbgl::util::readFile(filePath.string());
if (!maybeImage) {
@@ -507,15 +515,15 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
// setStyle
assert(operationArray.Size() >= 2u);
if (operationArray[1].IsString()) {
- std::string stylePath = localizeURL(operationArray[1].GetString(), testRootPath);
+ std::string stylePath = manifest.localizeURL(operationArray[1].GetString());
auto maybeStyle = readJson(stylePath);
if (maybeStyle.is<mbgl::JSDocument>()) {
auto& style = maybeStyle.get<mbgl::JSDocument>();
- localizeStyleURLs((mbgl::JSValue&)style, style, testRootPath);
+ manifest.localizeStyleURLs((mbgl::JSValue&)style, style);
map.getStyle().loadJSON(serializeJsonValue(style));
}
} else {
- localizeStyleURLs(operationArray[1], metadata.document, testRootPath);
+ manifest.localizeStyleURLs(operationArray[1], metadata.document);
map.getStyle().loadJSON(serializeJsonValue(operationArray[1]));
}
} else if (operationArray[0].GetString() == setCenterOp) {
@@ -616,7 +624,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
assert(operationArray[1].IsString());
assert(operationArray[2].IsObject());
- localizeSourceURLs(operationArray[2], metadata.document, testRootPath);
+ manifest.localizeSourceURLs(operationArray[2], metadata.document);
mbgl::style::conversion::Error error;
auto converted = mbgl::style::conversion::convert<std::unique_ptr<mbgl::style::Source>>(operationArray[2], error, operationArray[1].GetString());
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index 3efd17bf1e..bc97f8300b 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -3,6 +3,8 @@
#include <mbgl/gfx/headless_frontend.hpp>
#include <mbgl/map/map.hpp>
+#include "manifest_parser.hpp"
+
#include <memory>
#include <string>
@@ -11,11 +13,14 @@ struct TestMetadata;
class TestRunner {
public:
- TestRunner() = default;
- explicit TestRunner(const std::string& testRootPath);
+ explicit TestRunner(Manifest);
bool run(TestMetadata&);
void reset();
+ // Manifest
+ const Manifest& getManifest() const;
+ void doShuffle(uint32_t seed);
+
private:
bool runOperations(const std::string& key, TestMetadata&);
bool checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
@@ -32,5 +37,5 @@ private:
mbgl::Map map;
};
std::unordered_map<std::string, std::unique_ptr<Impl>> maps;
- std::string testRootPath{TEST_RUNNER_ROOT_PATH};
+ Manifest manifest;
};