summaryrefslogtreecommitdiff
path: root/render-test
diff options
context:
space:
mode:
authorMikhail Pozdnyakov <mikhail.pozdnyakov@mapbox.com>2019-11-27 17:26:44 +0200
committerMikhail Pozdnyakov <mikhail.pozdnyakov@mapbox.com>2019-11-27 18:19:10 +0200
commit2b6a0e48ee19913cde3b5737ee6a12fbf136cfe9 (patch)
treefeb136c57aec7ba91d1474c11bd4c86b9c21e2b8 /render-test
parentc35f64a2b55a8110b13a9a8d218bc3baa048d92e (diff)
downloadqtlocation-mapboxgl-2b6a0e48ee19913cde3b5737ee6a12fbf136cfe9.tar.gz
[test runner] Enable test results update from cmd line
Added the following command line argument: ``` -u[update], --update=[update] Test results update mode. Supported values are: "default", "platform", "metrics" ```
Diffstat (limited to 'render-test')
-rw-r--r--render-test/render_test.cpp38
-rw-r--r--render-test/runner.cpp13
-rw-r--r--render-test/runner.hpp4
3 files changed, 45 insertions, 10 deletions
diff --git a/render-test/render_test.cpp b/render-test/render_test.cpp
index 4d18902083..b0b2c745ec 100644
--- a/render-test/render_test.cpp
+++ b/render-test/render_test.cpp
@@ -40,7 +40,30 @@ void operator delete(void* ptr, size_t) noexcept {
#endif
namespace {
-using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>, std::string>;
+
+template <typename T>
+TestRunner::UpdateResults initUpdateResults(T& testUpdateResultsValue) {
+ if (!testUpdateResultsValue) {
+#if !TEST_READ_ONLY
+ if (getenv("UPDATE_DEFAULT")) return TestRunner::UpdateResults::DEFAULT;
+ if (getenv("UPDATE_PLATFORM")) return TestRunner::UpdateResults::PLATFORM;
+ if (getenv("UPDATE_METRICS")) return TestRunner::UpdateResults::METRICS;
+#endif
+ return TestRunner::UpdateResults::NO;
+ }
+ std::string str = args::get(testUpdateResultsValue);
+#if !TEST_READ_ONLY
+ if (str == "default") return TestRunner::UpdateResults::DEFAULT;
+ if (str == "platform") return TestRunner::UpdateResults::PLATFORM;
+ if (str == "metrics") return TestRunner::UpdateResults::METRICS;
+#endif
+ mbgl::Log::Warning(
+ mbgl::Event::General, "Unsupported update test results mode: \"%s\" will be ignored", str.c_str());
+ return TestRunner::UpdateResults::NO;
+}
+
+using ArgumentsTuple =
+ std::tuple<bool, bool, uint32_t, std::string, TestRunner::UpdateResults, std::vector<std::string>, std::string>;
ArgumentsTuple parseArguments(int argc, char** argv) {
args::ArgumentParser argumentParser("Mapbox GL Test Runner");
@@ -52,6 +75,11 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
args::ValueFlag<std::string> testPathValue(
argumentParser, "manifestPath", "Test manifest file path", {'p', "manifestPath"});
args::ValueFlag<std::string> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"});
+ args::ValueFlag<std::string> testUpdateResultsValue(
+ argumentParser,
+ "update",
+ "Test results update mode. Supported values are: \"default\", \"platform\", \"metrics\"",
+ {'u', "update"});
args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
try {
@@ -90,10 +118,12 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
auto testFilter = testFilterValue ? args::get(testFilterValue) : std::string{};
const auto shuffle = shuffleFlag ? args::get(shuffleFlag) : false;
const auto seed = seedValue ? args::get(seedValue) : 1u;
+ TestRunner::UpdateResults updateResults = initUpdateResults(testUpdateResultsValue);
return ArgumentsTuple{recycleMapFlag ? args::get(recycleMapFlag) : false,
shuffle,
seed,
manifestPath.string(),
+ updateResults,
std::move(testNames),
std::move(testFilter)};
}
@@ -107,14 +137,16 @@ int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
std::string manifestPath;
std::vector<std::string> testNames;
std::string testFilter;
+ TestRunner::UpdateResults updateResults;
- std::tie(recycleMap, shuffle, seed, manifestPath, testNames, testFilter) = parseArguments(argc, argv);
+ std::tie(recycleMap, shuffle, seed, manifestPath, updateResults, testNames, testFilter) =
+ parseArguments(argc, argv);
auto manifestData = ManifestParser::parseManifest(manifestPath, testNames, testFilter);
if (!manifestData) {
exit(5);
}
mbgl::util::RunLoop runLoop;
- TestRunner runner(std::move(*manifestData));
+ TestRunner runner(std::move(*manifestData), updateResults);
if (shuffle) {
printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed);
runner.doShuffle(seed);
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index c58095dd14..05ba089d81 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -154,7 +154,8 @@ std::string simpleDiff(const Value& result, const Value& expected) {
return diff.str();
}
-TestRunner::TestRunner(Manifest manifest_) : manifest(std::move(manifest_)) {}
+TestRunner::TestRunner(Manifest manifest_, UpdateResults updateResults_)
+ : manifest(std::move(manifest_)), updateResults(updateResults_) {}
const Manifest& TestRunner::getManifest() const {
return manifest;
@@ -185,11 +186,11 @@ bool TestRunner::checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
}
#if !TEST_READ_ONLY
- if (getenv("UPDATE_PLATFORM")) {
+ if (updateResults == UpdateResults::PLATFORM) {
mbgl::filesystem::create_directories(expectations.back());
mbgl::util::write_file(expectations.back().string() + "/expected.json", metadata.actualJson);
return true;
- } else if (getenv("UPDATE_DEFAULT")) {
+ } else if (updateResults == UpdateResults::DEFAULT) {
mbgl::util::write_file(base + "/expected.json", metadata.actualJson);
return true;
}
@@ -255,11 +256,11 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage,
}
#if !TEST_READ_ONLY
- if (getenv("UPDATE_PLATFORM")) {
+ if (updateResults == UpdateResults::PLATFORM) {
mbgl::filesystem::create_directories(expectations.back());
mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage));
return true;
- } else if (getenv("UPDATE_DEFAULT")) {
+ } else if (updateResults == UpdateResults::DEFAULT) {
mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage));
return true;
}
@@ -322,7 +323,7 @@ bool TestRunner::checkProbingResults(TestMetadata& metadata) {
if (metadata.metrics.isEmpty()) return true;
const std::vector<mbgl::filesystem::path>& expectedMetrics = metadata.paths.expectedMetrics;
#if !TEST_READ_ONLY
- if (getenv("UPDATE_METRICS")) {
+ if (updateResults == UpdateResults::METRICS) {
mbgl::filesystem::create_directories(expectedMetrics.back());
mbgl::util::write_file(expectedMetrics.back().string() + "/metrics.json", serializeMetrics(metadata.metrics));
return true;
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index 2503372602..5214f203d8 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -14,7 +14,8 @@ struct TestMetadata;
class TestRunner {
public:
- explicit TestRunner(Manifest);
+ enum class UpdateResults { NO, DEFAULT, PLATFORM, METRICS };
+ TestRunner(Manifest, UpdateResults);
bool run(TestMetadata&);
void reset();
@@ -43,4 +44,5 @@ private:
};
std::unordered_map<std::string, std::unique_ptr<Impl>> maps;
Manifest manifest;
+ UpdateResults updateResults;
};